diff --git a/library/core/src/ffi/c_str.rs b/library/core/src/ffi/c_str.rs index 4ea5cbf862645..b57ead374720e 100644 --- a/library/core/src/ffi/c_str.rs +++ b/library/core/src/ffi/c_str.rs @@ -3,11 +3,12 @@ use crate::cmp::Ordering; use crate::error::Error; use crate::ffi::c_char; +use crate::intrinsics::const_eval_select; use crate::iter::FusedIterator; use crate::marker::PhantomData; use crate::ptr::NonNull; use crate::slice::memchr; -use crate::{fmt, intrinsics, ops, slice, str}; +use crate::{fmt, ops, slice, str}; // FIXME: because this is doc(inline)d, we *have* to use intra-doc links because the actual link // depends on where the item is being documented. however, since this is libcore, we can't @@ -411,37 +412,35 @@ impl CStr { #[rustc_const_stable(feature = "const_cstr_unchecked", since = "1.59.0")] #[rustc_allow_const_fn_unstable(const_eval_select)] pub const unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr { - #[inline] - fn rt_impl(bytes: &[u8]) -> &CStr { - // Chance at catching some UB at runtime with debug builds. - debug_assert!(!bytes.is_empty() && bytes[bytes.len() - 1] == 0); - - // SAFETY: Casting to CStr is safe because its internal representation - // is a [u8] too (safe only inside std). - // Dereferencing the obtained pointer is safe because it comes from a - // reference. Making a reference is then safe because its lifetime - // is bound by the lifetime of the given `bytes`. - unsafe { &*(bytes as *const [u8] as *const CStr) } - } - - const fn const_impl(bytes: &[u8]) -> &CStr { - // Saturating so that an empty slice panics in the assert with a good - // message, not here due to underflow. - let mut i = bytes.len().saturating_sub(1); - assert!(!bytes.is_empty() && bytes[i] == 0, "input was not nul-terminated"); - - // Ending nul byte exists, skip to the rest. - while i != 0 { - i -= 1; - let byte = bytes[i]; - assert!(byte != 0, "input contained interior nul"); + const_eval_select!( + (bytes: &[u8]) -> &CStr: + if const { + // Saturating so that an empty slice panics in the assert with a good + // message, not here due to underflow. + let mut i = bytes.len().saturating_sub(1); + assert!(!bytes.is_empty() && bytes[i] == 0, "input was not nul-terminated"); + + // Ending nul byte exists, skip to the rest. + while i != 0 { + i -= 1; + let byte = bytes[i]; + assert!(byte != 0, "input contained interior nul"); + } + + // SAFETY: See runtime cast comment below. + unsafe { &*(bytes as *const [u8] as *const CStr) } + } else #[inline] { + // Chance at catching some UB at runtime with debug builds. + debug_assert!(!bytes.is_empty() && bytes[bytes.len() - 1] == 0); + + // SAFETY: Casting to CStr is safe because its internal representation + // is a [u8] too (safe only inside std). + // Dereferencing the obtained pointer is safe because it comes from a + // reference. Making a reference is then safe because its lifetime + // is bound by the lifetime of the given `bytes`. + unsafe { &*(bytes as *const [u8] as *const CStr) } } - - // SAFETY: See `rt_impl` cast. - unsafe { &*(bytes as *const [u8] as *const CStr) } - } - - intrinsics::const_eval_select((bytes,), const_impl, rt_impl) + ) } /// Returns the inner pointer to this C string. @@ -735,29 +734,27 @@ impl AsRef for CStr { #[cfg_attr(bootstrap, rustc_const_stable(feature = "const_cstr_from_ptr", since = "1.81.0"))] #[rustc_allow_const_fn_unstable(const_eval_select)] const unsafe fn strlen(ptr: *const c_char) -> usize { - const fn strlen_ct(s: *const c_char) -> usize { - let mut len = 0; - - // SAFETY: Outer caller has provided a pointer to a valid C string. - while unsafe { *s.add(len) } != 0 { - len += 1; - } + const_eval_select!( + (s: *const c_char = ptr) -> usize: + if const { + let mut len = 0; + + // SAFETY: Outer caller has provided a pointer to a valid C string. + while unsafe { *s.add(len) } != 0 { + len += 1; + } - len - } + len + } else #[inline] { + extern "C" { + /// Provided by libc or compiler_builtins. + fn strlen(s: *const c_char) -> usize; + } - #[inline] - fn strlen_rt(s: *const c_char) -> usize { - extern "C" { - /// Provided by libc or compiler_builtins. - fn strlen(s: *const c_char) -> usize; + // SAFETY: Outer caller has provided a pointer to a valid C string. + unsafe { strlen(s) } } - - // SAFETY: Outer caller has provided a pointer to a valid C string. - unsafe { strlen(s) } - } - - intrinsics::const_eval_select((ptr,), strlen_ct, strlen_rt) + ) } /// An iterator over the bytes of a [`CStr`], without the nul terminator. diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index fc09da7bcbc65..d82d911426427 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -2788,6 +2788,65 @@ where unreachable!() } +/// A macro to make it easier to invoke const_eval_select. Use as follows: +/// ```rust,ignore (just a macro example) +/// const_eval_select!( +/// #[inline] +/// (arg1: i32 = some_expr, arg2: T = other_expr) -> U: +/// if const { +/// // Compile-time code goes here. +/// } else { +/// // Run-time code goes here. +/// } +/// ) +/// ``` +pub(crate) macro const_eval_select { + ( + $(#[$attr:meta])* + ($($arg:ident : $ty:ty = $val:expr),* $(,)?) $( -> $ret:ty )?: + if const + $(#[$compiletime_attr:meta])* $compiletime:block + else + $(#[$runtime_attr:meta])* $runtime:block + ) => {{ + $(#[$attr])* + $(#[$runtime_attr])* + fn runtime($($arg: $ty),*) $( -> $ret )? { + $runtime + } + + $(#[$attr])* + $(#[$compiletime_attr])* + const fn compiletime($($arg: $ty),*) $( -> $ret )? { + // Don't warn if one of the arguments is unused. + $(let _ = $arg;)* + + $compiletime + } + + const_eval_select(($($val,)*), compiletime, runtime) + }}, + // We support leaving away the `val` expressions for *all* arguments + // (but not for *some* arguments, that's too tricky). + ( + $(#[$attr:meta])* + ($($arg:ident : $ty:ty),* $(,)?) -> $ret:ty: + if const + $(#[$compiletime_attr:meta])* $compiletime:block + else + $(#[$runtime_attr:meta])* $runtime:block + ) => { + $crate::intrinsics::const_eval_select!( + $(#[$attr])* + ($($arg : $ty = $arg),*) -> $ret: + if const + $(#[$compiletime_attr])* $compiletime + else + $(#[$runtime_attr])* $runtime + ) + }, +} + /// Returns whether the argument's value is statically known at /// compile-time. /// @@ -2830,7 +2889,7 @@ where /// # Stability concerns /// /// While it is safe to call, this intrinsic may behave differently in -/// a `const` context than otherwise. See the [`const_eval_select`] +/// a `const` context than otherwise. See the [`const_eval_select()`] /// documentation for an explanation of the issues this can cause. Unlike /// `const_eval_select`, this intrinsic isn't guaranteed to behave /// deterministically even in a `const` context. @@ -3734,14 +3793,15 @@ pub(crate) const fn miri_promise_symbolic_alignment(ptr: *const (), align: usize fn miri_promise_symbolic_alignment(ptr: *const (), align: usize); } - fn runtime(ptr: *const (), align: usize) { - // SAFETY: this call is always safe. - unsafe { - miri_promise_symbolic_alignment(ptr, align); + const_eval_select!( + (ptr: *const (), align: usize): + if const { + // Do nothing. + } else { + // SAFETY: this call is always safe. + unsafe { + miri_promise_symbolic_alignment(ptr, align); + } } - } - - const fn compiletime(_ptr: *const (), _align: usize) {} - - const_eval_select((ptr, align), compiletime, runtime); + ) } diff --git a/library/core/src/macros/mod.rs b/library/core/src/macros/mod.rs index 9a91ff82acd7c..c0bf798fb7069 100644 --- a/library/core/src/macros/mod.rs +++ b/library/core/src/macros/mod.rs @@ -24,18 +24,6 @@ macro_rules! panic { #[doc(hidden)] pub macro const_panic { ($const_msg:literal, $runtime_msg:literal, $($arg:ident : $ty:ty = $val:expr),* $(,)?) => {{ - #[inline] - #[track_caller] - fn runtime($($arg: $ty),*) -> ! { - $crate::panic!($runtime_msg); - } - - #[inline] - #[track_caller] - const fn compiletime($(_: $ty),*) -> ! { - $crate::panic!($const_msg); - } - // Wrap call to `const_eval_select` in a function so that we can // add the `rustc_allow_const_fn_unstable`. This is okay to do // because both variants will panic, just with different messages. @@ -44,7 +32,16 @@ pub macro const_panic { #[track_caller] #[cfg_attr(bootstrap, rustc_const_stable(feature = "const_panic", since = "CURRENT_RUSTC_VERSION"))] const fn do_panic($($arg: $ty),*) -> ! { - $crate::intrinsics::const_eval_select(($($arg),* ,), compiletime, runtime) + $crate::intrinsics::const_eval_select!( + #[inline] + #[track_caller] + ($($arg: $ty),*) -> !: + if const { + $crate::panic!($const_msg) + } else { + $crate::panic!($runtime_msg) + } + ) } do_panic($($val),*) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index a4e8e373e041e..7c5fa6b4e9072 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -33,26 +33,24 @@ impl *const T { #[rustc_diagnostic_item = "ptr_const_is_null"] #[inline] pub const fn is_null(self) -> bool { - #[inline] - fn runtime_impl(ptr: *const u8) -> bool { - ptr.addr() == 0 - } - - #[inline] - #[rustc_const_unstable(feature = "const_ptr_is_null", issue = "74939")] - const fn const_impl(ptr: *const u8) -> bool { - match (ptr).guaranteed_eq(null_mut()) { - Some(res) => res, - // To remain maximally convervative, we stop execution when we don't - // know whether the pointer is null or not. - // We can *not* return `false` here, that would be unsound in `NonNull::new`! - None => panic!("null-ness of this pointer cannot be determined in const context"), - } - } - // Compare via a cast to a thin pointer, so fat pointers are only // considering their "data" part for null-ness. - const_eval_select((self as *const u8,), const_impl, runtime_impl) + let ptr = self as *const u8; + const_eval_select!( + #[inline] + (ptr: *const u8) -> bool: + if const #[rustc_const_unstable(feature = "const_ptr_is_null", issue = "74939")] { + match (ptr).guaranteed_eq(null_mut()) { + Some(res) => res, + // To remain maximally convervative, we stop execution when we don't + // know whether the pointer is null or not. + // We can *not* return `false` here, that would be unsound in `NonNull::new`! + None => panic!("null-ness of this pointer cannot be determined in const context"), + } + } else { + ptr.addr() == 0 + } + ) } /// Casts to a pointer of another type. @@ -410,22 +408,20 @@ impl *const T { #[inline] #[rustc_allow_const_fn_unstable(const_eval_select)] const fn runtime_offset_nowrap(this: *const (), count: isize, size: usize) -> bool { - #[inline] - fn runtime(this: *const (), count: isize, size: usize) -> bool { - // We know `size <= isize::MAX` so the `as` cast here is not lossy. - let Some(byte_offset) = count.checked_mul(size as isize) else { - return false; - }; - let (_, overflow) = this.addr().overflowing_add_signed(byte_offset); - !overflow - } - - const fn comptime(_: *const (), _: isize, _: usize) -> bool { - true - } - // We can use const_eval_select here because this is only for UB checks. - intrinsics::const_eval_select((this, count, size), comptime, runtime) + const_eval_select!( + (this: *const (), count: isize, size: usize) -> bool: + if const { + true + } else #[inline] { + // We know `size <= isize::MAX` so the `as` cast here is not lossy. + let Some(byte_offset) = count.checked_mul(size as isize) else { + return false; + }; + let (_, overflow) = this.addr().overflowing_add_signed(byte_offset); + !overflow + } + ) } ub_checks::assert_unsafe_precondition!( @@ -763,14 +759,14 @@ impl *const T { { #[rustc_allow_const_fn_unstable(const_eval_select)] const fn runtime_ptr_ge(this: *const (), origin: *const ()) -> bool { - fn runtime(this: *const (), origin: *const ()) -> bool { - this >= origin - } - const fn comptime(_: *const (), _: *const ()) -> bool { - true - } - - intrinsics::const_eval_select((this, origin), comptime, runtime) + const_eval_select!( + (this: *const (), origin: *const ()) -> bool: + if const { + true + } else { + this >= origin + } + ) } ub_checks::assert_unsafe_precondition!( @@ -924,20 +920,18 @@ impl *const T { #[inline] #[rustc_allow_const_fn_unstable(const_eval_select)] const fn runtime_add_nowrap(this: *const (), count: usize, size: usize) -> bool { - #[inline] - fn runtime(this: *const (), count: usize, size: usize) -> bool { - let Some(byte_offset) = count.checked_mul(size) else { - return false; - }; - let (_, overflow) = this.addr().overflowing_add(byte_offset); - byte_offset <= (isize::MAX as usize) && !overflow - } - - const fn comptime(_: *const (), _: usize, _: usize) -> bool { - true - } - - intrinsics::const_eval_select((this, count, size), comptime, runtime) + const_eval_select!( + (this: *const (), count: usize, size: usize) -> bool: + if const { + true + } else #[inline] { + let Some(byte_offset) = count.checked_mul(size) else { + return false; + }; + let (_, overflow) = this.addr().overflowing_add(byte_offset); + byte_offset <= (isize::MAX as usize) && !overflow + } + ) } #[cfg(debug_assertions)] // Expensive, and doesn't catch much in the wild. @@ -1033,19 +1027,17 @@ impl *const T { #[inline] #[rustc_allow_const_fn_unstable(const_eval_select)] const fn runtime_sub_nowrap(this: *const (), count: usize, size: usize) -> bool { - #[inline] - fn runtime(this: *const (), count: usize, size: usize) -> bool { - let Some(byte_offset) = count.checked_mul(size) else { - return false; - }; - byte_offset <= (isize::MAX as usize) && this.addr() >= byte_offset - } - - const fn comptime(_: *const (), _: usize, _: usize) -> bool { - true - } - - intrinsics::const_eval_select((this, count, size), comptime, runtime) + const_eval_select!( + (this: *const (), count: usize, size: usize) -> bool: + if const { + true + } else #[inline] { + let Some(byte_offset) = count.checked_mul(size) else { + return false; + }; + byte_offset <= (isize::MAX as usize) && this.addr() >= byte_offset + } + ) } #[cfg(debug_assertions)] // Expensive, and doesn't catch much in the wild. diff --git a/library/core/src/ub_checks.rs b/library/core/src/ub_checks.rs index dd1454f401eaf..972d9ae26d2bf 100644 --- a/library/core/src/ub_checks.rs +++ b/library/core/src/ub_checks.rs @@ -95,20 +95,19 @@ pub use intrinsics::ub_checks as check_library_ub; #[inline] #[rustc_allow_const_fn_unstable(const_eval_select)] pub(crate) const fn check_language_ub() -> bool { - #[inline] - fn runtime() -> bool { - // Disable UB checks in Miri. - !cfg!(miri) - } - - #[inline] - const fn comptime() -> bool { - // Always disable UB checks. - false - } - // Only used for UB checks so we may const_eval_select. - intrinsics::ub_checks() && const_eval_select((), comptime, runtime) + intrinsics::ub_checks() + && const_eval_select!( + #[inline] + () -> bool: + if const { + // Always disable UB checks. + false + } else { + // Disable UB checks in Miri. + !cfg!(miri) + } + ) } /// Checks whether `ptr` is properly aligned with respect to the given alignment, and @@ -154,26 +153,24 @@ pub(crate) const fn is_nonoverlapping( size: usize, count: usize, ) -> bool { - #[inline] - fn runtime(src: *const (), dst: *const (), size: usize, count: usize) -> bool { - let src_usize = src.addr(); - let dst_usize = dst.addr(); - let Some(size) = size.checked_mul(count) else { - crate::panicking::panic_nounwind( - "is_nonoverlapping: `size_of::() * count` overflows a usize", - ) - }; - let diff = src_usize.abs_diff(dst_usize); - // If the absolute distance between the ptrs is at least as big as the size of the buffer, - // they do not overlap. - diff >= size - } - - #[inline] - const fn comptime(_: *const (), _: *const (), _: usize, _: usize) -> bool { - true - } - // This is just for safety checks so we can const_eval_select. - const_eval_select((src, dst, size, count), comptime, runtime) + const_eval_select!( + #[inline] + (src: *const (), dst: *const (), size: usize, count: usize) -> bool: + if const { + true + } else { + let src_usize = src.addr(); + let dst_usize = dst.addr(); + let Some(size) = size.checked_mul(count) else { + crate::panicking::panic_nounwind( + "is_nonoverlapping: `size_of::() * count` overflows a usize", + ) + }; + let diff = src_usize.abs_diff(dst_usize); + // If the absolute distance between the ptrs is at least as big as the size of the buffer, + // they do not overlap. + diff >= size + } + ) }