From 0f55179087253a374f4327985f24c0e1301c348b Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Thu, 23 Jun 2022 21:34:21 +0900 Subject: [PATCH] Allow CAS failure ordering stronger than success ordering --- CHANGELOG.md | 2 + src/imp/atomic128/aarch64.rs | 4 +- src/imp/atomic128/powerpc64.rs | 5 +- src/imp/atomic128/x86_64.rs | 3 + src/imp/core_atomic.rs | 99 ++++++++++++++++++++++++ src/lib.rs | 36 +++------ src/tests/helper.rs | 62 +++++++-------- src/utils.rs | 40 ++++++---- tests/cortex-m/src/main.rs | 135 +++++++++++++++++++++------------ 9 files changed, 266 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa65a8e..874cd996 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com ## [Unreleased] +- Allow CAS failure ordering stronger than success ordering. + ## [0.3.2] - 2022-06-19 - Optimize x86_64 128-bit atomic load/store on Intel CPU with AVX. ([#16](https://github.com/taiki-e/portable-atomic/pull/16)) diff --git a/src/imp/atomic128/aarch64.rs b/src/imp/atomic128/aarch64.rs index ba94746a..c8d2e876 100644 --- a/src/imp/atomic128/aarch64.rs +++ b/src/imp/atomic128/aarch64.rs @@ -325,8 +325,10 @@ unsafe fn atomic_compare_exchange( old: u128, new: u128, success: Ordering, - _failure: Ordering, + failure: Ordering, ) -> Result { + let success = crate::utils::upgrade_success_ordering(success, failure); + #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))] // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`. // cfg guarantee that the CPU supports FEAT_LSE. diff --git a/src/imp/atomic128/powerpc64.rs b/src/imp/atomic128/powerpc64.rs index 855207b7..d9aa4644 100644 --- a/src/imp/atomic128/powerpc64.rs +++ b/src/imp/atomic128/powerpc64.rs @@ -153,8 +153,11 @@ unsafe fn atomic_compare_exchange( old: u128, new: u128, success: Ordering, - _failure: Ordering, + failure: Ordering, ) -> Result { + debug_assert!(dst as usize % 16 == 0); + let success = crate::utils::upgrade_success_ordering(success, failure); + // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`. let res = unsafe { let old = U128 { whole: old }; diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs index 049c3c3a..4c0e6950 100644 --- a/src/imp/atomic128/x86_64.rs +++ b/src/imp/atomic128/x86_64.rs @@ -154,6 +154,7 @@ unsafe fn cmpxchg16b( #[inline] unsafe fn _atomic_load_vmovdqa(src: *mut u128, _order: Ordering) -> u128 { debug_assert!(src as usize % 16 == 0); + // SAFETY: the caller must uphold the safety contract for `_atomic_load_vmovdqa`. unsafe { let out: core::arch::x86_64::__m128; @@ -170,6 +171,7 @@ unsafe fn _atomic_load_vmovdqa(src: *mut u128, _order: Ordering) -> u128 { #[inline] unsafe fn _atomic_store_vmovdqa(dst: *mut u128, val: u128, _order: Ordering) { debug_assert!(dst as usize % 16 == 0); + // SAFETY: the caller must uphold the safety contract for `_atomic_store_vmovdqa`. unsafe { let val: core::arch::x86_64::__m128 = core::mem::transmute(val); @@ -274,6 +276,7 @@ unsafe fn atomic_compare_exchange( success: Ordering, failure: Ordering, ) -> Result { + let success = crate::utils::upgrade_success_ordering(success, failure); // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`. let (res, ok) = unsafe { cmpxchg16b(dst, old, new, success, failure) }; if ok { diff --git a/src/imp/core_atomic.rs b/src/imp/core_atomic.rs index 5a6dc15e..394df545 100644 --- a/src/imp/core_atomic.rs +++ b/src/imp/core_atomic.rs @@ -1,4 +1,7 @@ // Wrap the standard library's atomic types in newtype. +// +// This is not a reexport, because we want to backport changes like +// https://github.com/rust-lang/rust/pull/98383 to old compilers. #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] @@ -25,6 +28,34 @@ impl AtomicBool { pub(crate) fn into_inner(self) -> bool { self.inner.into_inner() } + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] + #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] + #[inline] + pub(crate) fn compare_exchange( + &self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering, + ) -> Result { + // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged. + let success = crate::utils::upgrade_success_ordering(success, failure); + self.inner.compare_exchange(current, new, success, failure) + } + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] + #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] + #[inline] + pub(crate) fn compare_exchange_weak( + &self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering, + ) -> Result { + // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged. + let success = crate::utils::upgrade_success_ordering(success, failure); + self.inner.compare_exchange_weak(current, new, success, failure) + } } impl core::ops::Deref for AtomicBool { type Target = core::sync::atomic::AtomicBool; @@ -61,6 +92,34 @@ impl AtomicPtr { pub(crate) fn into_inner(self) -> *mut T { self.inner.into_inner() } + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] + #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] + #[inline] + pub(crate) fn compare_exchange( + &self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering, + ) -> Result<*mut T, *mut T> { + // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged. + let success = crate::utils::upgrade_success_ordering(success, failure); + self.inner.compare_exchange(current, new, success, failure) + } + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] + #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] + #[inline] + pub(crate) fn compare_exchange_weak( + &self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering, + ) -> Result<*mut T, *mut T> { + // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged. + let success = crate::utils::upgrade_success_ordering(success, failure); + self.inner.compare_exchange_weak(current, new, success, failure) + } } impl core::ops::Deref for AtomicPtr { type Target = core::sync::atomic::AtomicPtr; @@ -99,6 +158,46 @@ macro_rules! atomic_int { pub(crate) fn into_inner(self) -> $int_type { self.inner.into_inner() } + #[cfg_attr( + portable_atomic_no_cfg_target_has_atomic, + cfg(not(portable_atomic_no_atomic_cas)) + )] + #[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(target_has_atomic = "ptr") + )] + #[inline] + pub(crate) fn compare_exchange( + &self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering, + ) -> Result<$int_type, $int_type> { + // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged. + let success = crate::utils::upgrade_success_ordering(success, failure); + self.inner.compare_exchange(current, new, success, failure) + } + #[cfg_attr( + portable_atomic_no_cfg_target_has_atomic, + cfg(not(portable_atomic_no_atomic_cas)) + )] + #[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(target_has_atomic = "ptr") + )] + #[inline] + pub(crate) fn compare_exchange_weak( + &self, + current: $int_type, + new: $int_type, + success: Ordering, + failure: Ordering, + ) -> Result<$int_type, $int_type> { + // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged. + let success = crate::utils::upgrade_success_ordering(success, failure); + self.inner.compare_exchange_weak(current, new, success, failure) + } #[cfg(portable_atomic_no_atomic_min_max)] #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, diff --git a/src/lib.rs b/src/lib.rs index f9cf44c9..4a38155d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -419,8 +419,7 @@ impl AtomicBool { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core)) @@ -454,8 +453,7 @@ impl AtomicBool { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core)) @@ -593,8 +591,7 @@ impl AtomicBool { /// Using [`Acquire`] as success ordering makes the store part of this /// operation [`Relaxed`], and using [`Release`] makes the final successful /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], - /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the - /// success ordering. + /// [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core)) @@ -792,8 +789,7 @@ impl AtomicPtr { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core)) @@ -827,8 +823,7 @@ impl AtomicPtr { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core)) @@ -867,8 +862,7 @@ impl AtomicPtr { /// Using [`Acquire`] as success ordering makes the store part of this /// operation [`Relaxed`], and using [`Release`] makes the final successful /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], - /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the - /// success ordering. + /// [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core)) @@ -1087,8 +1081,7 @@ atomic instructions or locks will be used. /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( @@ -1126,8 +1119,7 @@ atomic instructions or locks will be used. /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( @@ -1322,8 +1314,7 @@ atomic instructions or locks will be used. /// /// Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( @@ -1585,8 +1576,7 @@ This type has the same in-memory representation as the underlying floating point /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( @@ -1632,8 +1622,7 @@ This type has the same in-memory representation as the underlying floating point /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( @@ -1734,8 +1723,7 @@ This type has the same in-memory representation as the underlying floating point /// /// Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. #[cfg_attr( portable_atomic_no_cfg_target_has_atomic, cfg(any( diff --git a/src/tests/helper.rs b/src/tests/helper.rs index 2978826d..2d597c6b 100644 --- a/src/tests/helper.rs +++ b/src/tests/helper.rs @@ -236,10 +236,13 @@ macro_rules! __test_atomic_int { test_compare_exchange_ordering(|success, failure| { a.compare_exchange(5, 5, success, failure) }); - assert_eq!(a.compare_exchange(5, 10, Ordering::Acquire, Ordering::Relaxed), Ok(5)); - assert_eq!(a.load(Ordering::Relaxed), 10); - assert_eq!(a.compare_exchange(6, 12, Ordering::SeqCst, Ordering::Acquire), Err(10)); - assert_eq!(a.load(Ordering::Relaxed), 10); + for (success, failure) in compare_exchange_orderings() { + let a = <$atomic_type>::new(5); + assert_eq!(a.compare_exchange(5, 10, success, failure), Ok(5)); + assert_eq!(a.load(Ordering::Relaxed), 10); + assert_eq!(a.compare_exchange(6, 12, success, failure), Err(10)); + assert_eq!(a.load(Ordering::Relaxed), 10); + } } #[test] fn compare_exchange_weak() { @@ -247,16 +250,19 @@ macro_rules! __test_atomic_int { test_compare_exchange_ordering(|success, failure| { a.compare_exchange_weak(4, 4, success, failure) }); - assert_eq!(a.compare_exchange_weak(6, 8, Ordering::SeqCst, Ordering::Acquire), Err(4)); - let mut old = a.load(Ordering::Relaxed); - loop { - let new = old * 2; - match a.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { - Ok(_) => break, - Err(x) => old = x, + for (success, failure) in compare_exchange_orderings() { + let a = <$atomic_type>::new(4); + assert_eq!(a.compare_exchange_weak(6, 8, success, failure), Err(4)); + let mut old = a.load(Ordering::Relaxed); + loop { + let new = old * 2; + match a.compare_exchange_weak(old, new, success, failure) { + Ok(_) => break, + Err(x) => old = x, + } } + assert_eq!(a.load(Ordering::Relaxed), 8); } - assert_eq!(a.load(Ordering::Relaxed), 8); } #[test] fn fetch_add() { @@ -1262,6 +1268,7 @@ macro_rules! test_atomic_ptr_pub { } // Asserts that `a` and `b` have performed equivalent operations. +#[cfg(feature = "float")] macro_rules! assert_float_op_eq { ($a:expr, $b:expr $(,)?) => {{ // See also: @@ -1317,7 +1324,7 @@ pub(crate) fn load_orderings() -> [Ordering; 3] { [Ordering::Relaxed, Ordering::Acquire, Ordering::SeqCst] } pub(crate) fn rand_load_ordering() -> Ordering { - load_orderings()[fastrand::usize(0..3)] + load_orderings()[fastrand::usize(0..load_orderings().len())] } pub(crate) fn test_load_ordering(f: impl Fn(Ordering) -> T) { for &order in &load_orderings() { @@ -1339,7 +1346,7 @@ pub(crate) fn store_orderings() -> [Ordering; 3] { [Ordering::Relaxed, Ordering::Release, Ordering::SeqCst] } pub(crate) fn rand_store_ordering() -> Ordering { - store_orderings()[fastrand::usize(0..3)] + store_orderings()[fastrand::usize(0..store_orderings().len())] } pub(crate) fn test_store_ordering(f: impl Fn(Ordering) -> T) { for &order in &store_orderings() { @@ -1357,22 +1364,27 @@ pub(crate) fn test_store_ordering(f: impl Fn(Ordering) -> T) ); } } -pub(crate) fn compare_exchange_orderings() -> [(Ordering, Ordering); 9] { - // https://github.com/rust-lang/rust/blob/1.61.0/library/core/tests/atomic.rs#L199 +pub(crate) fn compare_exchange_orderings() -> [(Ordering, Ordering); 15] { [ (Ordering::Relaxed, Ordering::Relaxed), + (Ordering::Relaxed, Ordering::Acquire), + (Ordering::Relaxed, Ordering::SeqCst), (Ordering::Acquire, Ordering::Relaxed), + (Ordering::Acquire, Ordering::Acquire), + (Ordering::Acquire, Ordering::SeqCst), (Ordering::Release, Ordering::Relaxed), + (Ordering::Release, Ordering::Acquire), + (Ordering::Release, Ordering::SeqCst), (Ordering::AcqRel, Ordering::Relaxed), - (Ordering::SeqCst, Ordering::Relaxed), - (Ordering::Acquire, Ordering::Acquire), (Ordering::AcqRel, Ordering::Acquire), + (Ordering::AcqRel, Ordering::SeqCst), + (Ordering::SeqCst, Ordering::Relaxed), (Ordering::SeqCst, Ordering::Acquire), (Ordering::SeqCst, Ordering::SeqCst), ] } pub(crate) fn rand_compare_exchange_ordering() -> (Ordering, Ordering) { - compare_exchange_orderings()[fastrand::usize(0..9)] + compare_exchange_orderings()[fastrand::usize(0..compare_exchange_orderings().len())] } pub(crate) fn test_compare_exchange_ordering( f: impl Fn(Ordering, Ordering) -> T, @@ -1398,18 +1410,6 @@ pub(crate) fn test_compare_exchange_ordering( msg ); } - for &(success, failure) in &[ - (Ordering::Relaxed, Ordering::SeqCst), - (Ordering::Relaxed, Ordering::Acquire), - (Ordering::Acquire, Ordering::SeqCst), - (Ordering::Release, Ordering::SeqCst), - (Ordering::AcqRel, Ordering::SeqCst), - ] { - assert_eq!( - assert_panic(|| f(success, failure)), - "a failure ordering can't be stronger than a success ordering" - ); - } } } pub(crate) fn swap_orderings() -> [Ordering; 5] { diff --git a/src/utils.rs b/src/utils.rs index 665f92dc..0eefd30c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -198,24 +198,34 @@ pub(crate) fn assert_store_ordering(order: Ordering) { } } -// https://github.com/rust-lang/rust/blob/1.61.0/library/core/src/sync/atomic.rs#L2628 +// https://github.com/rust-lang/rust/pull/98383 #[inline] pub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) { - use core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst}; - #[allow(clippy::unnested_or_patterns)] + match success { + Ordering::AcqRel + | Ordering::Acquire + | Ordering::Relaxed + | Ordering::Release + | Ordering::SeqCst => {} + _ => unreachable!("{:?}", success), + } + match failure { + Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {} + Ordering::Release => panic!("there is no such thing as a release failure ordering"), + Ordering::AcqRel => panic!("there is no such thing as an acquire/release failure ordering"), + _ => unreachable!("{:?}", failure), + } +} + +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html +#[allow(dead_code)] +#[inline] +pub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering { match (success, failure) { - (Acquire, Acquire) - | (Release, Relaxed) - | (AcqRel, Acquire) - | (Relaxed, Relaxed) - | (SeqCst, SeqCst) - | (Acquire, Relaxed) - | (AcqRel, Relaxed) - | (SeqCst, Relaxed) - | (SeqCst, Acquire) => {} - (_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"), - (_, Release) => panic!("there is no such thing as a release failure ordering"), - _ => panic!("a failure ordering can't be stronger than a success ordering"), + (Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire, + (Ordering::Release, Ordering::Acquire) => Ordering::AcqRel, + (_, Ordering::SeqCst) => Ordering::SeqCst, + _ => success, } } diff --git a/tests/cortex-m/src/main.rs b/tests/cortex-m/src/main.rs index b0badb71..a6455641 100644 --- a/tests/cortex-m/src/main.rs +++ b/tests/cortex-m/src/main.rs @@ -36,83 +36,103 @@ macro_rules! __test_atomic { } compare_exchange(); fn compare_exchange() { - let a = <$atomic_type>::new(5); - assert_eq!(a.compare_exchange(5, 10, Ordering::Acquire, Ordering::Relaxed), Ok(5)); - assert_eq!(a.load(Ordering::Relaxed), 10); - assert_eq!(a.compare_exchange(6, 12, Ordering::SeqCst, Ordering::Acquire), Err(10)); - assert_eq!(a.load(Ordering::Relaxed), 10); + for (success, failure) in compare_exchange_orderings() { + let a = <$atomic_type>::new(5); + assert_eq!(a.compare_exchange(5, 10, success, failure), Ok(5)); + assert_eq!(a.load(Ordering::Relaxed), 10); + assert_eq!(a.compare_exchange(6, 12, success, failure), Err(10)); + assert_eq!(a.load(Ordering::Relaxed), 10); + } } compare_exchange_weak(); fn compare_exchange_weak() { - let a = <$atomic_type>::new(4); - assert_eq!(a.compare_exchange_weak(6, 8, Ordering::SeqCst, Ordering::Acquire), Err(4)); - let mut old = a.load(Ordering::Relaxed); - loop { - let new = old * 2; - match a.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) { - Ok(_) => break, - Err(x) => old = x, + for (success, failure) in compare_exchange_orderings() { + let a = <$atomic_type>::new(4); + assert_eq!(a.compare_exchange_weak(6, 8, success, failure), Err(4)); + let mut old = a.load(Ordering::Relaxed); + loop { + let new = old * 2; + match a.compare_exchange_weak(old, new, success, failure) { + Ok(_) => break, + Err(x) => old = x, + } } + assert_eq!(a.load(Ordering::Relaxed), 8); } - assert_eq!(a.load(Ordering::Relaxed), 8); } fetch_add(); fn fetch_add() { - let a = <$atomic_type>::new(0); - assert_eq!(a.fetch_add(10, Ordering::SeqCst), 0); - assert_eq!(a.load(Ordering::SeqCst), 10); - let a = <$atomic_type>::new($int_type::MAX); - assert_eq!(a.fetch_add(1, Ordering::SeqCst), $int_type::MAX); - assert_eq!(a.load(Ordering::SeqCst), $int_type::MAX.wrapping_add(1)); + for order in swap_orderings() { + let a = <$atomic_type>::new(0); + assert_eq!(a.fetch_add(10, order), 0); + assert_eq!(a.load(Ordering::SeqCst), 10); + let a = <$atomic_type>::new($int_type::MAX); + assert_eq!(a.fetch_add(1, order), $int_type::MAX); + assert_eq!(a.load(Ordering::SeqCst), $int_type::MAX.wrapping_add(1)); + } } fetch_sub(); fn fetch_sub() { - let a = <$atomic_type>::new(20); - assert_eq!(a.fetch_sub(10, Ordering::SeqCst), 20); - assert_eq!(a.load(Ordering::SeqCst), 10); - let a = <$atomic_type>::new($int_type::MIN); - assert_eq!(a.fetch_sub(1, Ordering::SeqCst), $int_type::MIN); - assert_eq!(a.load(Ordering::SeqCst), $int_type::MIN.wrapping_sub(1)); + for order in swap_orderings() { + let a = <$atomic_type>::new(20); + assert_eq!(a.fetch_sub(10, order), 20); + assert_eq!(a.load(Ordering::SeqCst), 10); + let a = <$atomic_type>::new($int_type::MIN); + assert_eq!(a.fetch_sub(1, order), $int_type::MIN); + assert_eq!(a.load(Ordering::SeqCst), $int_type::MIN.wrapping_sub(1)); + } } fetch_and(); fn fetch_and() { - let a = <$atomic_type>::new(0b101101); - assert_eq!(a.fetch_and(0b110011, Ordering::SeqCst), 0b101101); - assert_eq!(a.load(Ordering::SeqCst), 0b100001); + for order in swap_orderings() { + let a = <$atomic_type>::new(0b101101); + assert_eq!(a.fetch_and(0b110011, order), 0b101101); + assert_eq!(a.load(Ordering::SeqCst), 0b100001); + } } fetch_nand(); fn fetch_nand() { - let a = <$atomic_type>::new(0x13); - assert_eq!(a.fetch_nand(0x31, Ordering::SeqCst), 0x13); - assert_eq!(a.load(Ordering::SeqCst), !(0x13 & 0x31)); + for order in swap_orderings() { + let a = <$atomic_type>::new(0x13); + assert_eq!(a.fetch_nand(0x31, order), 0x13); + assert_eq!(a.load(Ordering::SeqCst), !(0x13 & 0x31)); + } } fetch_or(); fn fetch_or() { - let a = <$atomic_type>::new(0b101101); - assert_eq!(a.fetch_or(0b110011, Ordering::SeqCst), 0b101101); - assert_eq!(a.load(Ordering::SeqCst), 0b111111); + for order in swap_orderings() { + let a = <$atomic_type>::new(0b101101); + assert_eq!(a.fetch_or(0b110011, order), 0b101101); + assert_eq!(a.load(Ordering::SeqCst), 0b111111); + } } fetch_xor(); fn fetch_xor() { - let a = <$atomic_type>::new(0b101101); - assert_eq!(a.fetch_xor(0b110011, Ordering::SeqCst), 0b101101); - assert_eq!(a.load(Ordering::SeqCst), 0b011110); + for order in swap_orderings() { + let a = <$atomic_type>::new(0b101101); + assert_eq!(a.fetch_xor(0b110011, order), 0b101101); + assert_eq!(a.load(Ordering::SeqCst), 0b011110); + } } fetch_max(); fn fetch_max() { - let a = <$atomic_type>::new(23); - assert_eq!(a.fetch_max(22, Ordering::SeqCst), 23); - assert_eq!(a.load(Ordering::SeqCst), 23); - assert_eq!(a.fetch_max(24, Ordering::SeqCst), 23); - assert_eq!(a.load(Ordering::SeqCst), 24); + for order in swap_orderings() { + let a = <$atomic_type>::new(23); + assert_eq!(a.fetch_max(22, order), 23); + assert_eq!(a.load(Ordering::SeqCst), 23); + assert_eq!(a.fetch_max(24, order), 23); + assert_eq!(a.load(Ordering::SeqCst), 24); + } } fetch_min(); fn fetch_min() { - let a = <$atomic_type>::new(23); - assert_eq!(a.fetch_min(24, Ordering::SeqCst), 23); - assert_eq!(a.load(Ordering::SeqCst), 23); - assert_eq!(a.fetch_min(22, Ordering::SeqCst), 23); - assert_eq!(a.load(Ordering::SeqCst), 22); + for order in swap_orderings() { + let a = <$atomic_type>::new(23); + assert_eq!(a.fetch_min(24, order), 23); + assert_eq!(a.load(Ordering::SeqCst), 23); + assert_eq!(a.fetch_min(22, order), 23); + assert_eq!(a.load(Ordering::SeqCst), 22); + } } }; } @@ -126,6 +146,25 @@ fn store_orderings() -> [Ordering; 3] { fn swap_orderings() -> [Ordering; 5] { [Ordering::Relaxed, Ordering::Release, Ordering::Acquire, Ordering::AcqRel, Ordering::SeqCst] } +fn compare_exchange_orderings() -> [(Ordering, Ordering); 15] { + [ + (Ordering::Relaxed, Ordering::Relaxed), + (Ordering::Relaxed, Ordering::Acquire), + (Ordering::Relaxed, Ordering::SeqCst), + (Ordering::Acquire, Ordering::Relaxed), + (Ordering::Acquire, Ordering::Acquire), + (Ordering::Acquire, Ordering::SeqCst), + (Ordering::Release, Ordering::Relaxed), + (Ordering::Release, Ordering::Acquire), + (Ordering::Release, Ordering::SeqCst), + (Ordering::AcqRel, Ordering::Relaxed), + (Ordering::AcqRel, Ordering::Acquire), + (Ordering::AcqRel, Ordering::SeqCst), + (Ordering::SeqCst, Ordering::Relaxed), + (Ordering::SeqCst, Ordering::Acquire), + (Ordering::SeqCst, Ordering::SeqCst), + ] +} #[entry] fn main() -> ! {