Skip to content

Commit

Permalink
riscv: Support run-time detection of Zacas
Browse files Browse the repository at this point in the history 
Currently disabled by default as it is experimental as documented.
  • Loading branch information
@taiki-e
taiki-e committed Sep 21, 2024
1 parent ae5e275 commit c95ffe5
Show file tree
Hide file tree
Showing 19 changed files with 1,275 additions and 484 deletions.
1 change: 1 addition & 0 deletions .github/.cspell/project-dictionary.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ selgr
seqlock
setb
sete
sifive
signedness
simavr
skiboot
Expand Down
27 changes: 22 additions & 5 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,21 +337,39 @@ jobs:
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_test_outline_atomics_detect_false
# outline-atomics is disabled by default on AArch64 musl with static linking
if: (matrix.target == '' && !contains(matrix.rust, 'i686') || startsWith(matrix.target, 'x86_64')) || startsWith(matrix.target, 'aarch64') && !(contains(matrix.target, '-musl') && matrix.flags == '') || startsWith(matrix.target, 'armv5te') || matrix.target == 'arm-linux-androideabi'
# outline-atomics is disabled by default on AArch64 musl with static linking and powerpc64
# outline-atomics is disabled by default on AArch64 musl with static linking, powerpc64, and RISC-V
# powerpc64le- (little-endian) is skipped because it is pwr8 by default
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-} --exclude api-test
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics
if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl') && matrix.flags == '' || startsWith(matrix.target, 'powerpc64-')
if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl') && matrix.flags == '' || startsWith(matrix.target, 'powerpc64-') || startsWith(matrix.target, 'riscv')
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-} --exclude api-test
env:
# Note: detect_false cfg is intended to make it easy for portable-atomic developers to
# test cases such as has_cmpxchg16b == false, has_lse == false,
# __kuser_helper_version < 5, etc., and is not a public API.
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics --cfg portable_atomic_test_outline_atomics_detect_false
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics --cfg portable_atomic_test_outline_atomics_detect_false
if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl') && matrix.flags == '' || startsWith(matrix.target, 'powerpc64-')
if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl') && matrix.flags == '' || startsWith(matrix.target, 'powerpc64-') || startsWith(matrix.target, 'riscv')
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-} --exclude api-test
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics
QEMU_CPU: power7 # no quadword-atomics
if: startsWith(matrix.target, 'powerpc64-')
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-} --exclude api-test
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics
QEMU_CPU: sifive-u34 # no zacas
if: startsWith(matrix.target, 'riscv32')
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-} --exclude api-test
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics
RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics
QEMU_CPU: sifive-u54 # no zacas
if: startsWith(matrix.target, 'riscv64')
# x86_64 +cmpxchg16b
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
env:
Expand Down Expand Up @@ -401,8 +419,7 @@ jobs:
env:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+experimental-zacas
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+experimental-zacas
# TODO: cranelift doesn't support cfg(target_feature): https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
if: startsWith(matrix.target, 'riscv') && !contains(matrix.flags, 'codegen-backend=cranelift')
if: startsWith(matrix.target, 'riscv')
# s390x z196 (arch9)
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
env:
Expand Down
17 changes: 10 additions & 7 deletions build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ fn main() {
// Custom cfgs set by build script. Not public API.
// grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\)?".*$//' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
"cargo:rustc-check-cfg=cfg(portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_llvm_15,portable_atomic_no_llvm_16,portable_atomic_no_llvm_18,portable_atomic_new_atomic_intrinsics,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)"
"cargo:rustc-check-cfg=cfg(portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_llvm_15,portable_atomic_no_llvm_16,portable_atomic_no_llvm_18,portable_atomic_no_llvm_19,portable_atomic_new_atomic_intrinsics,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)"
);
// TODO: handle multi-line target_feature_fallback
// grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
Expand Down Expand Up @@ -167,12 +167,15 @@ fn main() {
println!("cargo:rustc-cfg=portable_atomic_no_atomic_load_store");
}

if version.llvm < 18 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_18");
if version.llvm < 16 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_16");
if version.llvm < 15 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_15");
if version.llvm < 19 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_19");
if version.llvm < 18 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_18");
if version.llvm < 16 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_16");
if version.llvm < 15 {
println!("cargo:rustc-cfg=portable_atomic_no_llvm_15");
}
}
}
}
Expand Down
166 changes: 82 additions & 84 deletions src/cfgs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,30 +158,29 @@ mod atomic_32_macros {
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
not(portable_atomic_no_asm),
not(portable_atomic_no_llvm_19),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
// TODO(riscv)
// all(
// feature = "fallback",
// not(portable_atomic_no_outline_atomics),
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
// any(
// all(
// target_os = "linux",
// any(
// target_env = "gnu",
// all(
// any(target_env = "musl", target_env = "ohos"),
// not(target_feature = "crt-static"),
// ),
// portable_atomic_outline_atomics,
// ),
// ),
// target_os = "android",
// ),
// not(any(miri, portable_atomic_sanitize_thread)),
// ),
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(
all(
target_os = "linux",
any(
target_env = "gnu",
all(
any(target_env = "musl", target_env = "ohos"),
not(target_feature = "crt-static"),
),
portable_atomic_outline_atomics,
),
),
target_os = "android",
),
),
),
),
))
Expand Down Expand Up @@ -235,30 +234,29 @@ mod atomic_64_macros {
target_arch = "riscv32",
not(any(miri, portable_atomic_sanitize_thread)),
not(portable_atomic_no_asm),
not(portable_atomic_no_llvm_19),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
// TODO(riscv)
// all(
// feature = "fallback",
// not(portable_atomic_no_outline_atomics),
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
// any(
// all(
// target_os = "linux",
// any(
// target_env = "gnu",
// all(
// any(target_env = "musl", target_env = "ohos"),
// not(target_feature = "crt-static"),
// ),
// portable_atomic_outline_atomics,
// ),
// ),
// target_os = "android",
// ),
// not(any(miri, portable_atomic_sanitize_thread)),
// ),
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(
all(
target_os = "linux",
any(
target_env = "gnu",
all(
any(target_env = "musl", target_env = "ohos"),
not(target_feature = "crt-static"),
),
portable_atomic_outline_atomics,
),
),
target_os = "android",
),
),
),
),
)))
Expand Down Expand Up @@ -304,30 +302,30 @@ mod atomic_64_macros {
all(
target_arch = "riscv64",
not(portable_atomic_no_asm),
not(portable_atomic_no_llvm_19),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
// TODO(riscv)
// all(
// feature = "fallback",
// not(portable_atomic_no_outline_atomics),
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
// any(
// all(
// target_os = "linux",
// any(
// target_env = "gnu",
// all(
// any(target_env = "musl", target_env = "ohos"),
// not(target_feature = "crt-static"),
// ),
// portable_atomic_outline_atomics,
// ),
// ),
// target_os = "android",
// ),
// not(any(miri, portable_atomic_sanitize_thread)),
// ),
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(
all(
target_os = "linux",
any(
target_env = "gnu",
all(
any(target_env = "musl", target_env = "ohos"),
not(target_feature = "crt-static"),
),
portable_atomic_outline_atomics,
),
),
target_os = "android",
),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
),
all(
Expand Down Expand Up @@ -423,30 +421,30 @@ mod atomic_128_macros {
all(
target_arch = "riscv64",
not(portable_atomic_no_asm),
not(portable_atomic_no_llvm_19),
any(
target_feature = "experimental-zacas",
portable_atomic_target_feature = "experimental-zacas",
// TODO(riscv)
// all(
// feature = "fallback",
// not(portable_atomic_no_outline_atomics),
// any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
// any(
// all(
// target_os = "linux",
// any(
// target_env = "gnu",
// all(
// any(target_env = "musl", target_env = "ohos"),
// not(target_feature = "crt-static"),
// ),
// portable_atomic_outline_atomics,
// ),
// ),
// target_os = "android",
// ),
// not(any(miri, portable_atomic_sanitize_thread)),
// ),
all(
feature = "fallback",
not(portable_atomic_no_outline_atomics),
any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
any(
all(
target_os = "linux",
any(
target_env = "gnu",
all(
any(target_env = "musl", target_env = "ohos"),
not(target_feature = "crt-static"),
),
portable_atomic_outline_atomics,
),
),
target_os = "android",
),
not(any(miri, portable_atomic_sanitize_thread)),
),
),
),
all(
Expand Down
4 changes: 2 additions & 2 deletions src/imp/atomic128/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u
| ----------- | ---- | ----- | --- | --- | ---- |
| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel, AMD, or Zhaoxin CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only. <br> Requires rustc 1.59+ |
| aarch64 | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires lse target feature, ldp/stp requires lse2 target feature, ldiapp/stilp requires lse2 and rcpc3 target features, swpp/ldclrp/ldsetp requires lse128 target feature. <br> Both compile-time and run-time detection are supported. <br> Requires rustc 1.59+ |
| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Experimental. Requires experimental-zacas target feature. Currently compile-time detection only due to LLVM marking it as experimental. <br> Requires 1.82+ (LLVM 19+) |
| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Experimental because LLVM marking the corresponding target feature as experimental. Requires experimental-zacas target feature. Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires 1.82+ (LLVM 19+) |
| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires target-cpu pwr8+ (powerpc64le is pwr8 by default). Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires nightly |
| s390x | lpq | stpq | cdsg | cdsg | Requires nightly |

Expand All @@ -20,7 +20,7 @@ See [aarch64.rs](aarch64.rs) module-level comments for more details on the instr

## Comparison with core::intrinsics::atomic_\* (core::sync::atomic::Atomic{I,U}128)

This directory has target-specific implementations with inline assembly [x86_64.rs](x86_64.rs), ([aarch64.rs](aarch64.rs), [riscv64.rs](riscv64.rs), [powerpc64.rs](powerpc64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
This directory has target-specific implementations with inline assembly ([x86_64.rs](x86_64.rs), [aarch64.rs](aarch64.rs), [riscv64.rs](riscv64.rs), [powerpc64.rs](powerpc64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.

Implementations with inline assembly generate assemblies almost equivalent to the `core::intrinsics::atomic_*` (used in `core::sync::atomic::Atomic{I,U}128`) for many operations, but some operations may or may not generate more efficient code. For example:

Expand Down
Loading

0 comments on commit c95ffe5

Please sign in to comment.