Skip to content

Commit

Permalink
Auto merge of #30969 - Amanieu:extended_atomic_cmpxchg, r=alexcrichton
Browse files Browse the repository at this point in the history
This is an implementation of rust-lang/rfcs#1443.
  • Loading branch information
bors committed Feb 22, 2016
2 parents d1f422e + 4fdbc2f commit d3929b2
Show file tree
Hide file tree
Showing 10 changed files with 577 additions and 49 deletions.
27 changes: 27 additions & 0 deletions src/libcore/intrinsics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,33 @@ extern "rust-intrinsic" {
pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;
pub fn atomic_cxchg_acqrel<T>(dst: *mut T, old: T, src: T) -> T;
pub fn atomic_cxchg_relaxed<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(not(stage0))]
pub fn atomic_cxchg_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(not(stage0))]
pub fn atomic_cxchg_failacq<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(not(stage0))]
pub fn atomic_cxchg_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(not(stage0))]
pub fn atomic_cxchg_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> T;

#[cfg(not(stage0))]
pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_acqrel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_relaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_failacq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(not(stage0))]
pub fn atomic_cxchgweak_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);

pub fn atomic_load<T>(src: *const T) -> T;
pub fn atomic_load_acq<T>(src: *const T) -> T;
Expand Down
435 changes: 426 additions & 9 deletions src/libcore/sync/atomic.rs

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion src/librustc_llvm/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1584,7 +1584,8 @@ extern {
CMP: ValueRef,
RHS: ValueRef,
Order: AtomicOrdering,
FailureOrder: AtomicOrdering)
FailureOrder: AtomicOrdering,
Weak: Bool)
-> ValueRef;
pub fn LLVMBuildAtomicRMW(B: BuilderRef,
Op: AtomicBinOp,
Expand Down
5 changes: 3 additions & 2 deletions src/librustc_trans/trans/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1067,8 +1067,9 @@ pub fn Resume(cx: Block, exn: ValueRef) -> ValueRef {
pub fn AtomicCmpXchg(cx: Block, dst: ValueRef,
cmp: ValueRef, src: ValueRef,
order: AtomicOrdering,
failure_order: AtomicOrdering) -> ValueRef {
B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order)
failure_order: AtomicOrdering,
weak: llvm::Bool) -> ValueRef {
B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order, weak)
}
pub fn AtomicRMW(cx: Block, op: AtomicBinOp,
dst: ValueRef, src: ValueRef,
Expand Down
5 changes: 3 additions & 2 deletions src/librustc_trans/trans/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1077,10 +1077,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
pub fn atomic_cmpxchg(&self, dst: ValueRef,
cmp: ValueRef, src: ValueRef,
order: AtomicOrdering,
failure_order: AtomicOrdering) -> ValueRef {
failure_order: AtomicOrdering,
weak: llvm::Bool) -> ValueRef {
unsafe {
llvm::LLVMBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src,
order, failure_order)
order, failure_order, weak)
}
}
pub fn atomic_rmw(&self, op: AtomicBinOp,
Expand Down
65 changes: 35 additions & 30 deletions src/librustc_trans/trans/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -678,49 +678,54 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
// "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
(_, name) if name.starts_with("atomic_") => {
let split: Vec<&str> = name.split('_').collect();
assert!(split.len() >= 2, "Atomic intrinsic not correct format");

let order = if split.len() == 2 {
llvm::SequentiallyConsistent
} else {
match split[2] {
"unordered" => llvm::Unordered,
"relaxed" => llvm::Monotonic,
"acq" => llvm::Acquire,
"rel" => llvm::Release,
"acqrel" => llvm::AcquireRelease,
let (order, failorder) = match split.len() {
2 => (llvm::SequentiallyConsistent, llvm::SequentiallyConsistent),
3 => match split[2] {
"unordered" => (llvm::Unordered, llvm::Unordered),
"relaxed" => (llvm::Monotonic, llvm::Monotonic),
"acq" => (llvm::Acquire, llvm::Acquire),
"rel" => (llvm::Release, llvm::Monotonic),
"acqrel" => (llvm::AcquireRelease, llvm::Acquire),
"failrelaxed" if split[1] == "cxchg" || split[1] == "cxchgweak" =>
(llvm::SequentiallyConsistent, llvm::Monotonic),
"failacq" if split[1] == "cxchg" || split[1] == "cxchgweak" =>
(llvm::SequentiallyConsistent, llvm::Acquire),
_ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
}
},
4 => match (split[2], split[3]) {
("acq", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" =>
(llvm::Acquire, llvm::Monotonic),
("acqrel", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" =>
(llvm::AcquireRelease, llvm::Monotonic),
_ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
},
_ => ccx.sess().fatal("Atomic intrinsic not in correct format"),
};

match split[1] {
"cxchg" => {
// See include/llvm/IR/Instructions.h for their implementation
// of this, I assume that it's good enough for us to use for
// now.
let strongest_failure_ordering = match order {
llvm::NotAtomic | llvm::Unordered =>
ccx.sess().fatal("cmpxchg must be atomic"),

llvm::Monotonic | llvm::Release =>
llvm::Monotonic,

llvm::Acquire | llvm::AcquireRelease =>
llvm::Acquire,

llvm::SequentiallyConsistent =>
llvm::SequentiallyConsistent
};

let tp_ty = *substs.types.get(FnSpace, 0);
let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
let src = from_arg_ty(bcx, llargs[2], tp_ty);
let res = AtomicCmpXchg(bcx, ptr, cmp, src, order,
strongest_failure_ordering);
let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::False);
ExtractValue(bcx, res, 0)
}

"cxchgweak" => {
let tp_ty = *substs.types.get(FnSpace, 0);
let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
let src = from_arg_ty(bcx, llargs[2], tp_ty);
let val = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::True);
let result = ExtractValue(bcx, val, 0);
let success = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx()));
Store(bcx, result, StructGEP(bcx, llresult, 0));
Store(bcx, success, StructGEP(bcx, llresult, 1));
C_nil(ccx)
}

"load" => {
let tp_ty = *substs.types.get(FnSpace, 0);
let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
Expand Down
4 changes: 4 additions & 0 deletions src/librustc_typeck/check/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &hir::ForeignItem) {
param(ccx, 0),
param(ccx, 0)),
param(ccx, 0)),
"cxchgweak" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)),
param(ccx, 0),
param(ccx, 0)),
tcx.mk_tup(vec!(param(ccx, 0), tcx.types.bool))),
"load" => (1, vec!(tcx.mk_imm_ptr(param(ccx, 0))),
param(ccx, 0)),
"store" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), param(ccx, 0)),
Expand Down
14 changes: 9 additions & 5 deletions src/rustllvm/RustWrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,15 @@ extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B,
LLVMValueRef old,
LLVMValueRef source,
AtomicOrdering order,
AtomicOrdering failure_order) {
return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(target), unwrap(old),
unwrap(source), order,
failure_order
));
AtomicOrdering failure_order,
LLVMBool weak) {
AtomicCmpXchgInst* acxi = unwrap(B)->CreateAtomicCmpXchg(unwrap(target),
unwrap(old),
unwrap(source),
order,
failure_order);
acxi->setWeak(weak);
return wrap(acxi);
}
extern "C" LLVMValueRef LLVMBuildAtomicFence(LLVMBuilderRef B,
AtomicOrdering order,
Expand Down
37 changes: 37 additions & 0 deletions src/test/run-pass/atomic-compare_exchange.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

#![feature(extended_compare_and_swap)]
use std::sync::atomic::{AtomicIsize, ATOMIC_ISIZE_INIT};
use std::sync::atomic::Ordering::*;

static ATOMIC: AtomicIsize = ATOMIC_ISIZE_INIT;

fn main() {
// Make sure trans can emit all the intrinsics correctly
ATOMIC.compare_exchange(0, 1, Relaxed, Relaxed);
ATOMIC.compare_exchange(0, 1, Acquire, Relaxed);
ATOMIC.compare_exchange(0, 1, Release, Relaxed);
ATOMIC.compare_exchange(0, 1, AcqRel, Relaxed);
ATOMIC.compare_exchange(0, 1, SeqCst, Relaxed);
ATOMIC.compare_exchange(0, 1, Acquire, Acquire);
ATOMIC.compare_exchange(0, 1, AcqRel, Acquire);
ATOMIC.compare_exchange(0, 1, SeqCst, Acquire);
ATOMIC.compare_exchange(0, 1, SeqCst, SeqCst);
ATOMIC.compare_exchange_weak(0, 1, Relaxed, Relaxed);
ATOMIC.compare_exchange_weak(0, 1, Acquire, Relaxed);
ATOMIC.compare_exchange_weak(0, 1, Release, Relaxed);
ATOMIC.compare_exchange_weak(0, 1, AcqRel, Relaxed);
ATOMIC.compare_exchange_weak(0, 1, SeqCst, Relaxed);
ATOMIC.compare_exchange_weak(0, 1, Acquire, Acquire);
ATOMIC.compare_exchange_weak(0, 1, AcqRel, Acquire);
ATOMIC.compare_exchange_weak(0, 1, SeqCst, Acquire);
ATOMIC.compare_exchange_weak(0, 1, SeqCst, SeqCst);
}
31 changes: 31 additions & 0 deletions src/test/run-pass/intrinsic-atomics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ mod rusti {
pub fn atomic_cxchg_acq<T>(dst: *mut T, old: T, src: T) -> T;
pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;

pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
pub fn atomic_cxchgweak_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
pub fn atomic_cxchgweak_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);

pub fn atomic_load<T>(src: *const T) -> T;
pub fn atomic_load_acq<T>(src: *const T) -> T;

Expand Down Expand Up @@ -79,5 +83,32 @@ pub fn main() {
assert_eq!(rusti::atomic_xsub_acq(&mut *x, 1), 2);
assert_eq!(rusti::atomic_xsub_rel(&mut *x, 1), 1);
assert_eq!(*x, 0);

loop {
let res = rusti::atomic_cxchgweak(&mut *x, 0, 1);
assert_eq!(res.0, 0);
if res.1 {
break;
}
}
assert_eq!(*x, 1);

loop {
let res = rusti::atomic_cxchgweak_acq(&mut *x, 1, 2);
assert_eq!(res.0, 1);
if res.1 {
break;
}
}
assert_eq!(*x, 2);

loop {
let res = rusti::atomic_cxchgweak_rel(&mut *x, 2, 3);
assert_eq!(res.0, 2);
if res.1 {
break;
}
}
assert_eq!(*x, 3);
}
}

0 comments on commit d3929b2

Please sign in to comment.