Auto merge of #30969 - Amanieu:extended_atomic_cmpxchg, r=alexcrichton

This is an implementation of rust-lang/rfcs#1443.

src/libcore/intrinsics.rs

@@ -58,6 +58,33 @@ extern "rust-intrinsic" {
     pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;
     pub fn atomic_cxchg_acqrel<T>(dst: *mut T, old: T, src: T) -> T;
     pub fn atomic_cxchg_relaxed<T>(dst: *mut T, old: T, src: T) -> T;
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchg_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchg_failacq<T>(dst: *mut T, old: T, src: T) -> T;
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchg_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchg_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
+
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_acqrel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_relaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_failacq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+    #[cfg(not(stage0))]
+    pub fn atomic_cxchgweak_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
 
     pub fn atomic_load<T>(src: *const T) -> T;
     pub fn atomic_load_acq<T>(src: *const T) -> T;

src/librustc_llvm/lib.rs

@@ -1584,7 +1584,8 @@ extern {
                                   CMP: ValueRef,
                                   RHS: ValueRef,
                                   Order: AtomicOrdering,
-                                  FailureOrder: AtomicOrdering)
+                                  FailureOrder: AtomicOrdering,
+                                  Weak: Bool)
                                   -> ValueRef;
     pub fn LLVMBuildAtomicRMW(B: BuilderRef,
                               Op: AtomicBinOp,

src/librustc_trans/trans/build.rs

@@ -1067,8 +1067,9 @@ pub fn Resume(cx: Block, exn: ValueRef) -> ValueRef {
 pub fn AtomicCmpXchg(cx: Block, dst: ValueRef,
                      cmp: ValueRef, src: ValueRef,
                      order: AtomicOrdering,
-                     failure_order: AtomicOrdering) -> ValueRef {
-    B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order)
+                     failure_order: AtomicOrdering,
+                     weak: llvm::Bool) -> ValueRef {
+    B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order, weak)
 }
 pub fn AtomicRMW(cx: Block, op: AtomicBinOp,
                  dst: ValueRef, src: ValueRef,

src/librustc_trans/trans/builder.rs

@@ -1077,10 +1077,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub fn atomic_cmpxchg(&self, dst: ValueRef,
                          cmp: ValueRef, src: ValueRef,
                          order: AtomicOrdering,
-                         failure_order: AtomicOrdering) -> ValueRef {
+                         failure_order: AtomicOrdering,
+                         weak: llvm::Bool) -> ValueRef {
         unsafe {
             llvm::LLVMBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src,
-                                         order, failure_order)
+                                         order, failure_order, weak)
         }
     }
     pub fn atomic_rmw(&self, op: AtomicBinOp,

src/librustc_trans/trans/intrinsic.rs

@@ -678,49 +678,54 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
         // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
         (_, name) if name.starts_with("atomic_") => {
             let split: Vec<&str> = name.split('_').collect();
-            assert!(split.len() >= 2, "Atomic intrinsic not correct format");
 
-            let order = if split.len() == 2 {
-                llvm::SequentiallyConsistent
-            } else {
-                match split[2] {
-                    "unordered" => llvm::Unordered,
-                    "relaxed" => llvm::Monotonic,
-                    "acq"     => llvm::Acquire,
-                    "rel"     => llvm::Release,
-                    "acqrel"  => llvm::AcquireRelease,
+            let (order, failorder) = match split.len() {
+                2 => (llvm::SequentiallyConsistent, llvm::SequentiallyConsistent),
+                3 => match split[2] {
+                    "unordered" => (llvm::Unordered, llvm::Unordered),
+                    "relaxed" => (llvm::Monotonic, llvm::Monotonic),
+                    "acq"     => (llvm::Acquire, llvm::Acquire),
+                    "rel"     => (llvm::Release, llvm::Monotonic),
+                    "acqrel"  => (llvm::AcquireRelease, llvm::Acquire),
+                    "failrelaxed" if split[1] == "cxchg" || split[1] == "cxchgweak" =>
+                        (llvm::SequentiallyConsistent, llvm::Monotonic),
+                    "failacq" if split[1] == "cxchg" || split[1] == "cxchgweak" =>
+                        (llvm::SequentiallyConsistent, llvm::Acquire),
                     _ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
-                }
+                },
+                4 => match (split[2], split[3]) {
+                    ("acq", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" =>
+                        (llvm::Acquire, llvm::Monotonic),
+                    ("acqrel", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" =>
+                        (llvm::AcquireRelease, llvm::Monotonic),
+                    _ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
+                },
+                _ => ccx.sess().fatal("Atomic intrinsic not in correct format"),
             };
 
             match split[1] {
                 "cxchg" => {
-                    // See include/llvm/IR/Instructions.h for their implementation
-                    // of this, I assume that it's good enough for us to use for
-                    // now.
-                    let strongest_failure_ordering = match order {
-                        llvm::NotAtomic | llvm::Unordered =>
-                            ccx.sess().fatal("cmpxchg must be atomic"),
-
-                        llvm::Monotonic | llvm::Release =>
-                            llvm::Monotonic,
-
-                        llvm::Acquire | llvm::AcquireRelease =>
-                            llvm::Acquire,
-
-                        llvm::SequentiallyConsistent =>
-                            llvm::SequentiallyConsistent
-                    };
-
                     let tp_ty = *substs.types.get(FnSpace, 0);
                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
                     let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
                     let src = from_arg_ty(bcx, llargs[2], tp_ty);
-                    let res = AtomicCmpXchg(bcx, ptr, cmp, src, order,
-                                            strongest_failure_ordering);
+                    let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::False);
                     ExtractValue(bcx, res, 0)
                 }
 
+                "cxchgweak" => {
+                    let tp_ty = *substs.types.get(FnSpace, 0);
+                    let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
+                    let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
+                    let src = from_arg_ty(bcx, llargs[2], tp_ty);
+                    let val = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::True);
+                    let result = ExtractValue(bcx, val, 0);
+                    let success = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx()));
+                    Store(bcx, result, StructGEP(bcx, llresult, 0));
+                    Store(bcx, success, StructGEP(bcx, llresult, 1));
+                    C_nil(ccx)
+                }
+
                 "load" => {
                     let tp_ty = *substs.types.get(FnSpace, 0);
                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);

src/librustc_typeck/check/intrinsic.rs

@@ -83,6 +83,10 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &hir::ForeignItem) {
                                 param(ccx, 0),
                                 param(ccx, 0)),
                         param(ccx, 0)),
+            "cxchgweak" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)),
+                                param(ccx, 0),
+                                param(ccx, 0)),
+                            tcx.mk_tup(vec!(param(ccx, 0), tcx.types.bool))),
             "load" => (1, vec!(tcx.mk_imm_ptr(param(ccx, 0))),
                        param(ccx, 0)),
             "store" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), param(ccx, 0)),

src/rustllvm/RustWrapper.cpp

@@ -191,11 +191,15 @@ extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B,
                                                LLVMValueRef old,
                                                LLVMValueRef source,
                                                AtomicOrdering order,
-                                               AtomicOrdering failure_order) {
-    return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(target), unwrap(old),
-                                               unwrap(source), order,
-                                               failure_order
-                                               ));
+                                               AtomicOrdering failure_order,
+                                               LLVMBool weak) {
+    AtomicCmpXchgInst* acxi = unwrap(B)->CreateAtomicCmpXchg(unwrap(target),
+                                                             unwrap(old),
+                                                             unwrap(source),
+                                                             order,
+                                                             failure_order);
+    acxi->setWeak(weak);
+    return wrap(acxi);
 }
 extern "C" LLVMValueRef LLVMBuildAtomicFence(LLVMBuilderRef B,
                                              AtomicOrdering order,

src/test/run-pass/atomic-compare_exchange.rs

@@ -0,0 +1,37 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(extended_compare_and_swap)]
+use std::sync::atomic::{AtomicIsize, ATOMIC_ISIZE_INIT};
+use std::sync::atomic::Ordering::*;
+
+static ATOMIC: AtomicIsize = ATOMIC_ISIZE_INIT;
+
+fn main() {
+    // Make sure trans can emit all the intrinsics correctly
+    ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed);
+    ATOMIC.compare_exchange(0, 1, Acquire, Relaxed);
+    ATOMIC.compare_exchange(0, 1, Release, Relaxed);
+    ATOMIC.compare_exchange(0, 1, AcqRel, Relaxed);
+    ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed);
+    ATOMIC.compare_exchange(0, 1, Acquire, Acquire);
+    ATOMIC.compare_exchange(0, 1, AcqRel, Acquire);
+    ATOMIC.compare_exchange(0, 1, SeqCst, Acquire);
+    ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst);
+    ATOMIC.compare_exchange_weak(0, 1, Relaxed, Relaxed);
+    ATOMIC.compare_exchange_weak(0, 1, Acquire, Relaxed);
+    ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed);
+    ATOMIC.compare_exchange_weak(0, 1, AcqRel, Relaxed);
+    ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed);
+    ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire);
+    ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire);
+    ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire);
+    ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst);
+}

src/test/run-pass/intrinsic-atomics.rs

@@ -19,6 +19,10 @@ mod rusti {
         pub fn atomic_cxchg_acq<T>(dst: *mut T, old: T, src: T) -> T;
         pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;
 
+        pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+        pub fn atomic_cxchgweak_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+        pub fn atomic_cxchgweak_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
+
         pub fn atomic_load<T>(src: *const T) -> T;
         pub fn atomic_load_acq<T>(src: *const T) -> T;
 
@@ -79,5 +83,32 @@ pub fn main() {
         assert_eq!(rusti::atomic_xsub_acq(&mut *x, 1), 2);
         assert_eq!(rusti::atomic_xsub_rel(&mut *x, 1), 1);
         assert_eq!(*x, 0);
+
+        loop {
+            let res = rusti::atomic_cxchgweak(&mut *x, 0, 1);
+            assert_eq!(res.0, 0);
+            if res.1 {
+                break;
+            }
+        }
+        assert_eq!(*x, 1);
+
+        loop {
+            let res = rusti::atomic_cxchgweak_acq(&mut *x, 1, 2);
+            assert_eq!(res.0, 1);
+            if res.1 {
+                break;
+            }
+        }
+        assert_eq!(*x, 2);
+
+        loop {
+            let res = rusti::atomic_cxchgweak_rel(&mut *x, 2, 3);
+            assert_eq!(res.0, 2);
+            if res.1 {
+                break;
+            }
+        }
+        assert_eq!(*x, 3);
     }
 }