Skip to content

Commit

Permalink
add tests and fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@vtjnash
vtjnash committed Apr 2, 2021
1 parent 298c8f5 commit f56ddb3
Show file tree
Hide file tree
Showing 6 changed files with 251 additions and 85 deletions.
18 changes: 17 additions & 1 deletion doc/src/base/multi-threading.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,25 @@ See also [Synchronization](@ref lib-task-sync).

## Atomic operations

```@docs
Base.@atomic
```

!!! note

The following APIs are fairly primitive, and will likely be exposed through an `unsafe_*`-like wrapper.

```
Core.Intrinsics.atomic_pointerref(pointer::Ptr{T}, order::Symbol) --> T
Core.Intrinsics.atomic_pointerset(pointer::Ptr{T}, new::T, order::Symbol) --> pointer
Core.Intrinsics.atomic_pointerswap(pointer::Ptr{T}, new::T, order::Symbol) --> old
Core.Intrinsics.atomic_pointermodify(pointer::Ptr{T}, function::(old::T,arg::S)->T, arg::S, order::Symbol) --> old
Core.Intrinsics.atomic_pointercmpswap(pointer::Ptr{T}, expected::Any, new::T, success_order::Symbol, failure_order::Symbol) --> (old, cmp)
```

!!! warning

The API for atomic operations has not yet been finalized and is likely to change.
The following APIs are deprecated, though support for them is likely to remain for several releases.

```@docs
Base.Threads.Atomic
Expand Down
5 changes: 4 additions & 1 deletion src/builtins.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -969,8 +969,11 @@ JL_CALLABLE(jl_f_cmpswapfield)
if (isatomic == (success_order == jl_memory_order_notatomic))
jl_atomic_error(isatomic ? "cmpswapfield!: atomic field cannot be written non-atomically"
: "cmpswapfield!: non-atomic field cannot be written atomically");
if (isatomic == (failure_order == jl_memory_order_notatomic))
jl_atomic_error(isatomic ? "cmpswapfield!: atomic field cannot be accessed non-atomically"
: "cmpswapfield!: non-atomic field cannot be accessed atomically");
if (failure_order > success_order)
jl_atomic_error("cmpswapfield!: invalid atomic ordering");
jl_atomic_error("invalid atomic ordering");
v = cmpswap_nth_field(st, v, idx, args[2], args[3], isatomic); // always seq_cst, if isatomic needed at all
return v;
}
Expand Down
122 changes: 76 additions & 46 deletions src/datatype.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -891,6 +891,10 @@ JL_DLLEXPORT int jl_atomic_bool_cmpswap_bits(char *dst, const jl_value_t *expect
// n.b.: this can spuriously fail if there are padding bits, the caller should deal with that
int success;
switch (nb) {
case 0: {
success = 1;
break;
}
case 1: {
uint8_t y = *(uint8_t*)expected;
success = jl_atomic_cmpswap((uint8_t*)dst, &y, *(uint8_t*)src);
Expand Down Expand Up @@ -941,53 +945,88 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, char *dst, co
jl_ptls_t ptls = jl_get_ptls_states();
jl_value_t *y = jl_gc_alloc(ptls, isptr ? nb : tuptyp->size, isptr ? dt : tuptyp);
int success;
jl_datatype_t *et = (jl_datatype_t*)jl_typeof(expected);
switch (nb) {
case 0: {
success = (dt == et);
break;
}
case 1: {
uint8_t *y8 = (uint8_t*)y;
*y8 = *(uint8_t*)expected;
success = jl_atomic_cmpswap((uint8_t*)dst, y8, *(uint8_t*)src);
if (dt == et) {
*y8 = *(uint8_t*)expected;
success = jl_atomic_cmpswap((uint8_t*)dst, y8, *(uint8_t*)src);
}
else {
*y8 = jl_atomic_load((uint8_t*)dst);
success = 0;
}
break;
}
case 2: {
uint16_t *y16 = (uint16_t*)y;
*y16 = *(uint16_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint16_t*)dst, y16, *(uint16_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
if (dt == et) {
*y16 = *(uint16_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint16_t*)dst, y16, *(uint16_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
}
}
else {
*y16 = jl_atomic_load((uint16_t*)dst);
success = 0;
}
break;
}
case 4: {
uint32_t *y32 = (uint32_t*)y;
*y32 = *(uint32_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint32_t*)dst, y32, *(uint32_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
if (dt == et) {
*y32 = *(uint32_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint32_t*)dst, y32, *(uint32_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
}
}
else {
*y32 = jl_atomic_load((uint32_t*)dst);
success = 0;
}
break;
}
#if MAX_POINTERATOMIC_SIZE > 4
case 8: {
uint64_t *y64 = (uint64_t*)y;
*y64 = *(uint64_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint64_t*)dst, y64, *(uint64_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
if (dt == et) {
*y64 = *(uint64_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint64_t*)dst, y64, *(uint64_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
}
}
else {
*y64 = jl_atomic_load((uint64_t*)dst);
success = 0;
}
break;
}
#endif
#if MAX_POINTERATOMIC_SIZE > 8
case 16: {
uint128_t *y128 = (uint128_t*)y;
*y128 = *(uint128_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint128_t*)dst, y128, *(uint128_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
if (dt == et) {
*y128 = *(uint128_t*)expected;
while (1) {
success = jl_atomic_cmpswap((uint128_t*)dst, y128, *(uint128_t*)src);
if (success || !dt->layout->haspadding || !jl_egal__bits(y, expected, dt))
break;
}
}
else {
*y128 = jl_atomic_load((uint128_t*)dst);
success = 0;
}
break;
}
Expand Down Expand Up @@ -1569,12 +1608,9 @@ jl_value_t *modify_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_valu
if (isunion) {
size_t fsz = jl_field_size(st, i);
uint8_t *psel = &((uint8_t*)v)[offs + fsz - 1];
unsigned nth = 0;
if (!jl_find_union_component(ty, jl_typeof(r), &nth))
assert(0 && "invalid field assignment to isbits union");
success = (*psel == nth);
success = (jl_typeof(r) == jl_nth_union_component(ty, *psel));
if (success) {
nth = 0;
unsigned nth = 0;
if (!jl_find_union_component(ty, yty, &nth))
assert(0 && "invalid field assignment to isbits union");
*psel = nth;
Expand Down Expand Up @@ -1627,7 +1663,6 @@ jl_value_t *cmpswap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_val
JL_GC_POP();
}
else {
jl_value_t *rty = jl_typeof(r);
int hasptr;
int isunion = jl_is_uniontype(ty);
if (isunion) {
Expand All @@ -1637,7 +1672,10 @@ jl_value_t *cmpswap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_val
else {
hasptr = ((jl_datatype_t*)ty)->layout->npointers > 0;
}
size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy
jl_value_t *rty = ty;
size_t fsz;
if (!isunion)
fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy
int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE);
if (isatomic && !needlock) {
r = jl_atomic_cmpswap_bits((jl_datatype_t*)rty, (char*)v + offs, r, rhs, fsz);
Expand All @@ -1646,45 +1684,37 @@ jl_value_t *cmpswap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_val
jl_gc_multi_wb(v, rhs); // rhs is immutable
}
else {
jl_ptls_t ptls = jl_get_ptls_states();
uint8_t *psel;
unsigned nth;
int success;
if (isunion) {
size_t fsz = jl_field_size(st, i);
psel = &((uint8_t*)v)[offs + fsz - 1];
nth = 0;
success = jl_find_union_component(ty, rty, &nth);
uint8_t sel = *psel;
if (success)
success = nth == sel;
rty = jl_nth_union_component(ty, sel);
}
else {
success = rty == ty;
rty = jl_nth_union_component(rty, *psel);
}
jl_value_t *params[2];
params[0] = rty;
params[1] = (jl_value_t*)jl_bool_type;
jl_datatype_t *tuptyp = jl_apply_tuple_type_v(params, 2);
JL_GC_PROMISE_ROOTED(tuptyp); // (JL_ALWAYS_LEAFTYPE)
assert(!jl_field_isptr(tuptyp, 0));
jl_ptls_t ptls = jl_get_ptls_states();
r = jl_gc_alloc(ptls, tuptyp->size, (jl_value_t*)tuptyp);
int success = (rty == jl_typeof(expected));
if (needlock)
jl_lock_value(v);
size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy
memcpy((char*)r, (char*)v + offs, fsz);
if (success) {
memcpy((char*)r, (char*)v + offs, fsz);
if (((jl_datatype_t*)rty)->layout->haspadding)
success = jl_egal__bits(r, expected, (jl_datatype_t*)rty);
else
success = memcmp((char*)r, (char*)expected, fsz) == 0;
}
*((uint8_t*)r + fsz) = success ? 1 : 0;
if (success) {
rty = jl_typeof(rhs);
fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy
jl_value_t *rty = jl_typeof(rhs);
size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy
if (isunion) {
nth = 0;
unsigned nth = 0;
if (!jl_find_union_component(ty, rty, &nth))
assert(0 && "invalid field assignment to isbits union");
*psel = nth;
Expand All @@ -1696,7 +1726,7 @@ jl_value_t *cmpswap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_val
if (needlock)
jl_unlock_value(v);
}
r = undefref_check((jl_datatype_t*)ty, r);
r = undefref_check((jl_datatype_t*)rty, r);
if (__unlikely(r == NULL))
jl_throw(jl_undefref_exception);
}
Expand Down
2 changes: 0 additions & 2 deletions src/runtime_intrinsics.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,6 @@ JL_DLLEXPORT jl_value_t *jl_atomic_pointercmpswap(jl_value_t *p, jl_value_t *exp
jl_error("pointercmpswap: invalid pointer");
if (jl_typeof(x) != ety)
jl_type_error("pointercmpswap", ety, x);
if (jl_typeof(expected) != ety)
jl_type_error("pointercmpswap", ety, expected);
size_t nb = jl_datatype_size(ety);
if ((nb & (nb - 1)) != 0 || nb > MAX_POINTERATOMIC_SIZE)
jl_error("pointercmpswap: invalid atomic operation");
Expand Down
Loading

0 comments on commit f56ddb3

Please sign in to comment.