diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 2e2292d63f4c6..47f5d68f31158 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -58,6 +58,33 @@ extern "rust-intrinsic" { pub fn atomic_cxchg_rel(dst: *mut T, old: T, src: T) -> T; pub fn atomic_cxchg_acqrel(dst: *mut T, old: T, src: T) -> T; pub fn atomic_cxchg_relaxed(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_failrelaxed(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_failacq(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_acq_failrelaxed(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_acqrel_failrelaxed(dst: *mut T, old: T, src: T) -> T; + + #[cfg(not(stage0))] + pub fn atomic_cxchgweak(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acq(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_rel(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acqrel(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_failacq(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acq_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acqrel_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); pub fn atomic_load(src: *const T) -> T; pub fn atomic_load_acq(src: *const T) -> T; diff --git a/src/libcore/sync/atomic.rs b/src/libcore/sync/atomic.rs index 700a577e20c92..0c831bff763d0 100644 --- a/src/libcore/sync/atomic.rs +++ b/src/libcore/sync/atomic.rs @@ -311,10 +311,101 @@ impl AtomicBool { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn compare_and_swap(&self, current: bool, new: bool, order: Ordering) -> bool { + self.compare_exchange(current, new, order, strongest_failure_ordering(order)) + } + + /// Stores a value into the `bool` if the current value is the same as the `current` value. + /// + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. + /// + /// `compare_exchange` takes two `Ordering` arguments to describe the memory ordering of this + /// operation. The first describes the required ordering if the operation succeeds while the + /// second describes the required ordering when the operation fails. The failure ordering can't + /// be `Acquire` or `AcqRel` and must be equivalent or weaker than the success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let some_bool = AtomicBool::new(true); + /// + /// assert_eq!(some_bool.compare_exchange(true, + /// false, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// true); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// + /// assert_eq!(some_bool.compare_exchange(true, true, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// false); + /// assert_eq!(some_bool.load(Ordering::Relaxed), false); + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange(&self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering) -> bool { let current = if current { UINT_TRUE } else { 0 }; let new = if new { UINT_TRUE } else { 0 }; - unsafe { atomic_compare_and_swap(self.v.get(), current, new, order) > 0 } + unsafe { atomic_compare_exchange(self.v.get(), current, new, success, failure) > 0 } + } + + /// Stores a value into the `bool` if the current value is the same as the `current` value. + /// + /// Unlike `compare_exchange`, this function is allowed to spuriously fail even when the + /// comparison succeeds, which can result in more efficient code on some platforms. The + /// returned value is a tuple of the existing value and a flag indicating whether the + /// new value was written. + /// + /// `compare_exchange_weak` takes two `Ordering` arguments to describe the memory + /// ordering of this operation. The first describes the required ordering if the operation + /// succeeds while the second describes the required ordering when the operation fails. The + /// failure ordering can't be `Acquire` or `AcqRel` and must be equivalent or weaker than the + /// success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicBool, Ordering}; + /// + /// let val = AtomicBool::new(false); + /// + /// let new = true; + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// let result = val.compare_exchange_weak(old, new, + /// Ordering::SeqCst, + /// Ordering::Relaxed); + /// if result.1 { + /// break; + /// } else { + /// old = result.0; + /// } + /// } + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange_weak(&self, + current: bool, + new: bool, + success: Ordering, + failure: Ordering) -> (bool, bool) { + let current = if current { UINT_TRUE } else { 0 }; + let new = if new { UINT_TRUE } else { 0 }; + + let result = unsafe { + atomic_compare_exchange_weak(self.v.get(), current, new, success, failure) + }; + (result.0 > 0, result.1) } /// Logical "and" with a boolean value. @@ -553,7 +644,91 @@ impl AtomicIsize { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn compare_and_swap(&self, current: isize, new: isize, order: Ordering) -> isize { - unsafe { atomic_compare_and_swap(self.v.get(), current, new, order) } + self.compare_exchange(current, new, order, strongest_failure_ordering(order)) + } + + /// Stores a value into the `isize` if the current value is the same as the `current` value. + /// + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. + /// + /// `compare_exchange` takes two `Ordering` arguments to describe the memory ordering of this + /// operation. The first describes the required ordering if the operation succeeds while the + /// second describes the required ordering when the operation fails. The failure ordering can't + /// be `Acquire` or `AcqRel` and must be equivalent or weaker than the success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicIsize, Ordering}; + /// + /// let some_isize = AtomicIsize::new(5); + /// + /// assert_eq!(some_isize.compare_exchange(5, 10, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// 5); + /// assert_eq!(some_isize.load(Ordering::Relaxed), 10); + /// + /// assert_eq!(some_isize.compare_exchange(6, 12, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// 10); + /// assert_eq!(some_isize.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange(&self, + current: isize, + new: isize, + success: Ordering, + failure: Ordering) -> isize { + unsafe { atomic_compare_exchange(self.v.get(), current, new, success, failure) } + } + + /// Stores a value into the `isize if the current value is the same as the `current` value. + /// + /// Unlike `compare_exchange`, this function is allowed to spuriously fail even when the + /// comparison succeeds, which can result in more efficient code on some platforms. The + /// returned value is a tuple of the existing value and a flag indicating whether the + /// new value was written. + /// + /// `compare_exchange_weak` takes two `Ordering` arguments to describe the memory + /// ordering of this operation. The first describes the required ordering if the operation + /// succeeds while the second describes the required ordering when the operation fails. The + /// failure ordering can't be `Acquire` or `AcqRel` and must be equivalent or weaker than the + /// success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicIsize, Ordering}; + /// + /// let val = AtomicIsize::new(4); + /// + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// let new = old * 2; + /// let result = val.compare_exchange_weak(old, new, + /// Ordering::SeqCst, + /// Ordering::Relaxed); + /// if result.1 { + /// break; + /// } else { + /// old = result.0; + /// } + /// } + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange_weak(&self, + current: isize, + new: isize, + success: Ordering, + failure: Ordering) -> (isize, bool) { + unsafe { atomic_compare_exchange_weak(self.v.get(), current, new, success, failure) } } /// Add an isize to the current value, returning the previous value. @@ -746,7 +921,91 @@ impl AtomicUsize { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn compare_and_swap(&self, current: usize, new: usize, order: Ordering) -> usize { - unsafe { atomic_compare_and_swap(self.v.get(), current, new, order) } + self.compare_exchange(current, new, order, strongest_failure_ordering(order)) + } + + /// Stores a value into the `usize` if the current value is the same as the `current` value. + /// + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. + /// + /// `compare_exchange` takes two `Ordering` arguments to describe the memory ordering of this + /// operation. The first describes the required ordering if the operation succeeds while the + /// second describes the required ordering when the operation fails. The failure ordering can't + /// be `Acquire` or `AcqRel` and must be equivalent or weaker than the success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicUsize, Ordering}; + /// + /// let some_isize = AtomicUsize::new(5); + /// + /// assert_eq!(some_isize.compare_exchange(5, 10, + /// Ordering::Acquire, + /// Ordering::Relaxed), + /// 5); + /// assert_eq!(some_isize.load(Ordering::Relaxed), 10); + /// + /// assert_eq!(some_isize.compare_exchange(6, 12, + /// Ordering::SeqCst, + /// Ordering::Acquire), + /// 10); + /// assert_eq!(some_isize.load(Ordering::Relaxed), 10); + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange(&self, + current: usize, + new: usize, + success: Ordering, + failure: Ordering) -> usize { + unsafe { atomic_compare_exchange(self.v.get(), current, new, success, failure) } + } + + /// Stores a value into the `usize` if the current value is the same as the `current` value. + /// + /// Unlike `compare_exchange`, this function is allowed to spuriously fail even when the + /// comparison succeeds, which can result in more efficient code on some platforms. The + /// returned value is a tuple of the existing value and a flag indicating whether the + /// new value was written. + /// + /// `compare_exchange_weak` takes two `Ordering` arguments to describe the memory + /// ordering of this operation. The first describes the required ordering if the operation + /// succeeds while the second describes the required ordering when the operation fails. The + /// failure ordering can't be `Acquire` or `AcqRel` and must be equivalent or weaker than the + /// success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicUsize, Ordering}; + /// + /// let val = AtomicUsize::new(4); + /// + /// let mut old = val.load(Ordering::Relaxed); + /// loop { + /// let new = old * 2; + /// let result = val.compare_exchange_weak(old, new, + /// Ordering::SeqCst, + /// Ordering::Relaxed); + /// if result.1 { + /// break; + /// } else { + /// old = result.0; + /// } + /// } + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange_weak(&self, + current: usize, + new: usize, + success: Ordering, + failure: Ordering) -> (usize, bool) { + unsafe { atomic_compare_exchange_weak(self.v.get(), current, new, success, failure) } } /// Add to the current usize, returning the previous value. @@ -947,15 +1206,109 @@ impl AtomicPtr { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn compare_and_swap(&self, current: *mut T, new: *mut T, order: Ordering) -> *mut T { + self.compare_exchange(current, new, order, strongest_failure_ordering(order)) + } + + /// Stores a value into the pointer if the current value is the same as the `current` value. + /// + /// The return value is always the previous value. If it is equal to `current`, then the value + /// was updated. + /// + /// `compare_exchange` takes two `Ordering` arguments to describe the memory ordering of this + /// operation. The first describes the required ordering if the operation succeeds while the + /// second describes the required ordering when the operation fails. The failure ordering can't + /// be `Acquire` or `AcqRel` and must be equivalent or weaker than the success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let ptr = &mut 5; + /// let some_ptr = AtomicPtr::new(ptr); + /// + /// let other_ptr = &mut 10; + /// let another_ptr = &mut 10; + /// + /// let value = some_ptr.compare_exchange(other_ptr, another_ptr, + /// Ordering::SeqCst, Ordering::Relaxed); + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange(&self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering) -> *mut T { unsafe { - atomic_compare_and_swap(self.p.get() as *mut usize, current as usize, - new as usize, order) as *mut T + atomic_compare_exchange(self.p.get() as *mut usize, current as usize, + new as usize, success, failure) as *mut T } } + + /// Stores a value into the pointer if the current value is the same as the `current` value. + /// + /// Unlike `compare_exchange`, this function is allowed to spuriously fail even when the + /// comparison succeeds, which can result in more efficient code on some platforms. The + /// returned value is a tuple of the existing value and a flag indicating whether the + /// new value was written. + /// + /// `compare_exchange_weak` takes two `Ordering` arguments to describe the memory + /// ordering of this operation. The first describes the required ordering if the operation + /// succeeds while the second describes the required ordering when the operation fails. The + /// failure ordering can't be `Acquire` or `AcqRel` and must be equivalent or weaker than the + /// success ordering. + /// + /// # Examples + /// + /// ``` + /// # #![feature(extended_compare_and_swap)] + /// use std::sync::atomic::{AtomicPtr, Ordering}; + /// + /// let some_ptr = AtomicPtr::new(&mut 5); + /// + /// let new = &mut 10; + /// let mut old = some_ptr.load(Ordering::Relaxed); + /// loop { + /// let result = some_ptr.compare_exchange_weak(old, new, + /// Ordering::SeqCst, + /// Ordering::Relaxed); + /// if result.1 { + /// break; + /// } else { + /// old = result.0; + /// } + /// } + /// ``` + #[inline] + #[unstable(feature = "extended_compare_and_swap", reason = "recently added", issue = "31767")] + pub fn compare_exchange_weak(&self, + current: *mut T, + new: *mut T, + success: Ordering, + failure: Ordering) -> (*mut T, bool) { + let result = unsafe { + atomic_compare_exchange_weak(self.p.get() as *mut usize, current as usize, + new as usize, success, failure) + }; + (result.0 as *mut T, result.1) + } } #[inline] -unsafe fn atomic_store(dst: *mut T, val: T, order:Ordering) { +fn strongest_failure_ordering(order: Ordering) -> Ordering { + match order { + Release => Relaxed, + Relaxed => Relaxed, + SeqCst => SeqCst, + Acquire => Acquire, + AcqRel => Acquire, + } +} + +#[inline] +unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { match order { Release => intrinsics::atomic_store_rel(dst, val), Relaxed => intrinsics::atomic_store_relaxed(dst, val), @@ -966,7 +1319,7 @@ unsafe fn atomic_store(dst: *mut T, val: T, order:Ordering) { } #[inline] -unsafe fn atomic_load(dst: *const T, order:Ordering) -> T { +unsafe fn atomic_load(dst: *const T, order: Ordering) -> T { match order { Acquire => intrinsics::atomic_load_acq(dst), Relaxed => intrinsics::atomic_load_relaxed(dst), @@ -1012,8 +1365,36 @@ unsafe fn atomic_sub(dst: *mut T, val: T, order: Ordering) -> T { } #[inline] -unsafe fn atomic_compare_and_swap(dst: *mut T, old:T, new:T, order: Ordering) -> T { - match order { +#[cfg(not(stage0))] +unsafe fn atomic_compare_exchange(dst: *mut T, + old: T, + new: T, + success: Ordering, + failure: Ordering) -> T { + match (success, failure) { + (Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new), + (Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new), + (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new), + (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new), + (SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new), + (Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new), + (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new), + (SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new), + (SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new), + (_, Release) => panic!("there is no such thing as an acquire/release failure ordering"), + (_, AcqRel) => panic!("there is no such thing as a release failure ordering"), + _ => panic!("a failure ordering can't be stronger than a success ordering"), + } +} + +#[inline] +#[cfg(stage0)] +unsafe fn atomic_compare_exchange(dst: *mut T, + old: T, + new: T, + success: Ordering, + _: Ordering) -> T { + match success { Acquire => intrinsics::atomic_cxchg_acq(dst, old, new), Release => intrinsics::atomic_cxchg_rel(dst, old, new), AcqRel => intrinsics::atomic_cxchg_acqrel(dst, old, new), @@ -1022,6 +1403,42 @@ unsafe fn atomic_compare_and_swap(dst: *mut T, old:T, new:T, order: Ordering) } } +#[inline] +#[cfg(not(stage0))] +unsafe fn atomic_compare_exchange_weak(dst: *mut T, + old: T, + new: T, + success: Ordering, + failure: Ordering) -> (T, bool) { + match (success, failure) { + (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new), + (Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new), + (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new), + (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new), + (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new), + (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new), + (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new), + (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new), + (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new), + (_, Release) => panic!("there is no such thing as an acquire/release failure ordering"), + (_, AcqRel) => panic!("there is no such thing as a release failure ordering"), + _ => panic!("a failure ordering can't be stronger than a success ordering"), + } +} + +#[inline] +#[cfg(stage0)] +unsafe fn atomic_compare_exchange_weak(dst: *mut T, + old: T, + new: T, + success: Ordering, + failure: Ordering) -> (T, bool) + where T: ::cmp::Eq + ::marker::Copy +{ + let result = atomic_compare_exchange(dst, old, new, success, failure); + (result, result == old) +} + #[inline] unsafe fn atomic_and(dst: *mut T, val: T, order: Ordering) -> T { match order { diff --git a/src/librustc_llvm/lib.rs b/src/librustc_llvm/lib.rs index dfaf39702372c..13acf79a0f1d0 100644 --- a/src/librustc_llvm/lib.rs +++ b/src/librustc_llvm/lib.rs @@ -1584,7 +1584,8 @@ extern { CMP: ValueRef, RHS: ValueRef, Order: AtomicOrdering, - FailureOrder: AtomicOrdering) + FailureOrder: AtomicOrdering, + Weak: Bool) -> ValueRef; pub fn LLVMBuildAtomicRMW(B: BuilderRef, Op: AtomicBinOp, diff --git a/src/librustc_trans/trans/build.rs b/src/librustc_trans/trans/build.rs index ce541c8d411bb..2b4fcf436cbc0 100644 --- a/src/librustc_trans/trans/build.rs +++ b/src/librustc_trans/trans/build.rs @@ -1067,8 +1067,9 @@ pub fn Resume(cx: Block, exn: ValueRef) -> ValueRef { pub fn AtomicCmpXchg(cx: Block, dst: ValueRef, cmp: ValueRef, src: ValueRef, order: AtomicOrdering, - failure_order: AtomicOrdering) -> ValueRef { - B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order) + failure_order: AtomicOrdering, + weak: llvm::Bool) -> ValueRef { + B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order, weak) } pub fn AtomicRMW(cx: Block, op: AtomicBinOp, dst: ValueRef, src: ValueRef, diff --git a/src/librustc_trans/trans/builder.rs b/src/librustc_trans/trans/builder.rs index 878d01f46b63f..434fca4168858 100644 --- a/src/librustc_trans/trans/builder.rs +++ b/src/librustc_trans/trans/builder.rs @@ -1077,10 +1077,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { pub fn atomic_cmpxchg(&self, dst: ValueRef, cmp: ValueRef, src: ValueRef, order: AtomicOrdering, - failure_order: AtomicOrdering) -> ValueRef { + failure_order: AtomicOrdering, + weak: llvm::Bool) -> ValueRef { unsafe { llvm::LLVMBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src, - order, failure_order) + order, failure_order, weak) } } pub fn atomic_rmw(&self, op: AtomicBinOp, diff --git a/src/librustc_trans/trans/intrinsic.rs b/src/librustc_trans/trans/intrinsic.rs index 188fb7de9dd83..b7b520f6c8259 100644 --- a/src/librustc_trans/trans/intrinsic.rs +++ b/src/librustc_trans/trans/intrinsic.rs @@ -678,49 +678,54 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, // "atomic_[_]", and no ordering means SeqCst (_, name) if name.starts_with("atomic_") => { let split: Vec<&str> = name.split('_').collect(); - assert!(split.len() >= 2, "Atomic intrinsic not correct format"); - let order = if split.len() == 2 { - llvm::SequentiallyConsistent - } else { - match split[2] { - "unordered" => llvm::Unordered, - "relaxed" => llvm::Monotonic, - "acq" => llvm::Acquire, - "rel" => llvm::Release, - "acqrel" => llvm::AcquireRelease, + let (order, failorder) = match split.len() { + 2 => (llvm::SequentiallyConsistent, llvm::SequentiallyConsistent), + 3 => match split[2] { + "unordered" => (llvm::Unordered, llvm::Unordered), + "relaxed" => (llvm::Monotonic, llvm::Monotonic), + "acq" => (llvm::Acquire, llvm::Acquire), + "rel" => (llvm::Release, llvm::Monotonic), + "acqrel" => (llvm::AcquireRelease, llvm::Acquire), + "failrelaxed" if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::SequentiallyConsistent, llvm::Monotonic), + "failacq" if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::SequentiallyConsistent, llvm::Acquire), _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") - } + }, + 4 => match (split[2], split[3]) { + ("acq", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::Acquire, llvm::Monotonic), + ("acqrel", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::AcquireRelease, llvm::Monotonic), + _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") + }, + _ => ccx.sess().fatal("Atomic intrinsic not in correct format"), }; match split[1] { "cxchg" => { - // See include/llvm/IR/Instructions.h for their implementation - // of this, I assume that it's good enough for us to use for - // now. - let strongest_failure_ordering = match order { - llvm::NotAtomic | llvm::Unordered => - ccx.sess().fatal("cmpxchg must be atomic"), - - llvm::Monotonic | llvm::Release => - llvm::Monotonic, - - llvm::Acquire | llvm::AcquireRelease => - llvm::Acquire, - - llvm::SequentiallyConsistent => - llvm::SequentiallyConsistent - }; - let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let cmp = from_arg_ty(bcx, llargs[1], tp_ty); let src = from_arg_ty(bcx, llargs[2], tp_ty); - let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, - strongest_failure_ordering); + let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::False); ExtractValue(bcx, res, 0) } + "cxchgweak" => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let cmp = from_arg_ty(bcx, llargs[1], tp_ty); + let src = from_arg_ty(bcx, llargs[2], tp_ty); + let val = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::True); + let result = ExtractValue(bcx, val, 0); + let success = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx())); + Store(bcx, result, StructGEP(bcx, llresult, 0)); + Store(bcx, success, StructGEP(bcx, llresult, 1)); + C_nil(ccx) + } + "load" => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index ba6fa9aed3d9a..5e1dc35870bb2 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -83,6 +83,10 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &hir::ForeignItem) { param(ccx, 0), param(ccx, 0)), param(ccx, 0)), + "cxchgweak" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), + param(ccx, 0), + param(ccx, 0)), + tcx.mk_tup(vec!(param(ccx, 0), tcx.types.bool))), "load" => (1, vec!(tcx.mk_imm_ptr(param(ccx, 0))), param(ccx, 0)), "store" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), param(ccx, 0)), diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index ecf9146a1d011..4ebe49512d74b 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -191,11 +191,15 @@ extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef old, LLVMValueRef source, AtomicOrdering order, - AtomicOrdering failure_order) { - return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(target), unwrap(old), - unwrap(source), order, - failure_order - )); + AtomicOrdering failure_order, + LLVMBool weak) { + AtomicCmpXchgInst* acxi = unwrap(B)->CreateAtomicCmpXchg(unwrap(target), + unwrap(old), + unwrap(source), + order, + failure_order); + acxi->setWeak(weak); + return wrap(acxi); } extern "C" LLVMValueRef LLVMBuildAtomicFence(LLVMBuilderRef B, AtomicOrdering order, diff --git a/src/test/run-pass/atomic-compare_exchange.rs b/src/test/run-pass/atomic-compare_exchange.rs new file mode 100644 index 0000000000000..7946704709ce4 --- /dev/null +++ b/src/test/run-pass/atomic-compare_exchange.rs @@ -0,0 +1,37 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(extended_compare_and_swap)] +use std::sync::atomic::{AtomicIsize, ATOMIC_ISIZE_INIT}; +use std::sync::atomic::Ordering::*; + +static ATOMIC: AtomicIsize = ATOMIC_ISIZE_INIT; + +fn main() { + // Make sure trans can emit all the intrinsics correctly + ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed); + ATOMIC.compare_exchange(0, 1, Acquire, Relaxed); + ATOMIC.compare_exchange(0, 1, Release, Relaxed); + ATOMIC.compare_exchange(0, 1, AcqRel, Relaxed); + ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed); + ATOMIC.compare_exchange(0, 1, Acquire, Acquire); + ATOMIC.compare_exchange(0, 1, AcqRel, Acquire); + ATOMIC.compare_exchange(0, 1, SeqCst, Acquire); + ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst); + ATOMIC.compare_exchange_weak(0, 1, Relaxed, Relaxed); + ATOMIC.compare_exchange_weak(0, 1, Acquire, Relaxed); + ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed); + ATOMIC.compare_exchange_weak(0, 1, AcqRel, Relaxed); + ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed); + ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire); + ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire); + ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire); + ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst); +} diff --git a/src/test/run-pass/intrinsic-atomics.rs b/src/test/run-pass/intrinsic-atomics.rs index 4ccab55e94348..3cc125e951334 100644 --- a/src/test/run-pass/intrinsic-atomics.rs +++ b/src/test/run-pass/intrinsic-atomics.rs @@ -19,6 +19,10 @@ mod rusti { pub fn atomic_cxchg_acq(dst: *mut T, old: T, src: T) -> T; pub fn atomic_cxchg_rel(dst: *mut T, old: T, src: T) -> T; + pub fn atomic_cxchgweak(dst: *mut T, old: T, src: T) -> (T, bool); + pub fn atomic_cxchgweak_acq(dst: *mut T, old: T, src: T) -> (T, bool); + pub fn atomic_cxchgweak_rel(dst: *mut T, old: T, src: T) -> (T, bool); + pub fn atomic_load(src: *const T) -> T; pub fn atomic_load_acq(src: *const T) -> T; @@ -79,5 +83,32 @@ pub fn main() { assert_eq!(rusti::atomic_xsub_acq(&mut *x, 1), 2); assert_eq!(rusti::atomic_xsub_rel(&mut *x, 1), 1); assert_eq!(*x, 0); + + loop { + let res = rusti::atomic_cxchgweak(&mut *x, 0, 1); + assert_eq!(res.0, 0); + if res.1 { + break; + } + } + assert_eq!(*x, 1); + + loop { + let res = rusti::atomic_cxchgweak_acq(&mut *x, 1, 2); + assert_eq!(res.0, 1); + if res.1 { + break; + } + } + assert_eq!(*x, 2); + + loop { + let res = rusti::atomic_cxchgweak_rel(&mut *x, 2, 3); + assert_eq!(res.0, 2); + if res.1 { + break; + } + } + assert_eq!(*x, 3); } }