From a7434da9be1fd77137e8f145d677c77dad2269e6 Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Wed, 22 Jun 2022 13:15:03 +0200 Subject: [PATCH 1/2] Remove restrictions on compare-exchange memory ordering. --- library/core/src/sync/atomic.rs | 53 +++++++++++++-------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index a68c6080e3a17..672db00565657 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -581,8 +581,7 @@ impl AtomicBool { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic /// operations on `u8`. @@ -640,8 +639,7 @@ impl AtomicBool { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic /// operations on `u8`. @@ -905,8 +903,7 @@ impl AtomicBool { /// Using [`Acquire`] as success ordering makes the store part of this /// operation [`Relaxed`], and using [`Release`] makes the final successful /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], - /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the - /// success ordering. + /// [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic /// operations on `u8`. @@ -1265,8 +1262,7 @@ impl AtomicPtr { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic /// operations on pointers. @@ -1311,8 +1307,7 @@ impl AtomicPtr { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic /// operations on pointers. @@ -1368,8 +1363,7 @@ impl AtomicPtr { /// Using [`Acquire`] as success ordering makes the store part of this /// operation [`Relaxed`], and using [`Release`] makes the final successful /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], - /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the - /// success ordering. + /// [`Acquire`] or [`Relaxed`]. /// /// **Note:** This method is only available on platforms that support atomic /// operations on pointers. @@ -1850,8 +1844,7 @@ macro_rules! atomic_int { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note**: This method is only available on platforms that support atomic operations on #[doc = concat!("[`", $s_int_type, "`].")] @@ -1902,8 +1895,7 @@ macro_rules! atomic_int { /// `failure` describes the required ordering for the load operation that takes place when /// the comparison fails. Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the successful load - /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note**: This method is only available on platforms that support atomic operations on #[doc = concat!("[`", $s_int_type, "`].")] @@ -2140,8 +2132,7 @@ macro_rules! atomic_int { /// /// Using [`Acquire`] as success ordering makes the store part /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load - /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`] - /// and must be equivalent to or weaker than the success ordering. + /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]. /// /// **Note**: This method is only available on platforms that support atomic operations on #[doc = concat!("[`", $s_int_type, "`].")] @@ -2658,23 +2649,22 @@ unsafe fn atomic_compare_exchange( let (val, ok) = unsafe { match (success, failure) { (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new), - //(Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new), - //(Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new), + (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new), + (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new), (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new), (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new), - //(Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new), + (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new), (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new), - //(Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new), - //(Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new), + (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new), + (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new), (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new), (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new), - //(AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new), + (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new), (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new), (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new), (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new), (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"), (_, Release) => panic!("there is no such thing as a release failure ordering"), - _ => panic!("a failure ordering can't be stronger than a success ordering"), } }; if ok { Ok(val) } else { Err(val) } @@ -2693,23 +2683,22 @@ unsafe fn atomic_compare_exchange_weak( let (val, ok) = unsafe { match (success, failure) { (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new), - //(Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new), - //(Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new), + (Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new), + (Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new), (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new), (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new), - //(Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new), + (Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new), (Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new), - //(Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new), - //(Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new), + (Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new), + (Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new), (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new), (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new), - //(AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new), + (AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new), (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new), (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new), (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new), (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"), (_, Release) => panic!("there is no such thing as a release failure ordering"), - _ => panic!("a failure ordering can't be stronger than a success ordering"), } }; if ok { Ok(val) } else { Err(val) } From a898f413795d923d1a025517458fa0fd2d7291ac Mon Sep 17 00:00:00 2001 From: Mara Bos Date: Wed, 22 Jun 2022 13:53:12 +0200 Subject: [PATCH 2/2] Only enable new cmpxchg memory orderings in cfg(not(bootstrap)). (The bootstrap/beta compiler doesn't support them yet.) --- library/core/src/sync/atomic.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 672db00565657..514c20c71eb46 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -2649,22 +2649,30 @@ unsafe fn atomic_compare_exchange( let (val, ok) = unsafe { match (success, failure) { (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new), + #[cfg(not(bootstrap))] (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new), + #[cfg(not(bootstrap))] (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new), (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new), (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new), + #[cfg(not(bootstrap))] (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new), (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new), + #[cfg(not(bootstrap))] (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new), + #[cfg(not(bootstrap))] (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new), (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new), (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new), + #[cfg(not(bootstrap))] (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new), (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new), (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new), (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new), (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"), (_, Release) => panic!("there is no such thing as a release failure ordering"), + #[cfg(bootstrap)] + _ => panic!("a failure ordering can't be stronger than a success ordering"), } }; if ok { Ok(val) } else { Err(val) } @@ -2683,22 +2691,30 @@ unsafe fn atomic_compare_exchange_weak( let (val, ok) = unsafe { match (success, failure) { (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new), + #[cfg(not(bootstrap))] (Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new), + #[cfg(not(bootstrap))] (Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new), (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new), (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new), + #[cfg(not(bootstrap))] (Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new), (Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new), + #[cfg(not(bootstrap))] (Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new), + #[cfg(not(bootstrap))] (Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new), (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new), (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new), + #[cfg(not(bootstrap))] (AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new), (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new), (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new), (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new), (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"), (_, Release) => panic!("there is no such thing as a release failure ordering"), + #[cfg(bootstrap)] + _ => panic!("a failure ordering can't be stronger than a success ordering"), } }; if ok { Ok(val) } else { Err(val) }