From 7c10562845682f605d361d1c7d0fbfa38180bc87 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 27 Aug 2024 08:04:41 -0700 Subject: [PATCH] Re-enable testing of WASI on CI This commit updates CI to resume testing WASI. This updates the container and testing scripts from historical processes to more modern ones, e.g. downloading wasi-sdk instead of compiling a custom toolchain. This should make it easier to update in the future and keep it in sync with rust-lang/rust as well. This also required a few minor fixes such as: * The `S_IFIFO` and `S_IFMT` constants had incorrect values. * The `CLOCK_*` definitions cause `ctest2`'s parsing to panic to they're skipped with a new `#[cfg]`. * A new `langinfo.h` header was added to the list to include. * Some historically skipped checks were removed since they're no longer necessary. * Checks for `__errno_location` are disabled since that doesn't actually exist in headers. * Checks for `select` are disabled because the Rust definition got the `const`-ness swapped for the final `timeval` argument. --- .github/workflows/full_ci.yml | 6 +---- build.rs | 1 + ci/docker/wasm32-wasi/Dockerfile | 40 ------------------------------ ci/docker/wasm32-wasi/clang.sh | 2 -- ci/docker/wasm32-wasip1/Dockerfile | 32 ++++++++++++++++++++++++ libc-test/build.rs | 35 ++++++++++++++++++-------- src/wasi.rs | 35 ++++++++++++++++---------- 7 files changed, 80 insertions(+), 71 deletions(-) delete mode 100644 ci/docker/wasm32-wasi/Dockerfile delete mode 100755 ci/docker/wasm32-wasi/clang.sh create mode 100644 ci/docker/wasm32-wasip1/Dockerfile diff --git a/.github/workflows/full_ci.yml b/.github/workflows/full_ci.yml index 042d81b158c0c..822dc5c716198 100644 --- a/.github/workflows/full_ci.yml +++ b/.github/workflows/full_ci.yml @@ -118,11 +118,7 @@ jobs: powerpc64le-unknown-linux-gnu, s390x-unknown-linux-gnu, riscv64gc-unknown-linux-gnu, - # FIXME: A recent nightly causes a linker failure: - # https://github.com/rust-lang/rust/issues/76679 - # See this comment for more details: - # https://github.com/rust-lang/libc/pull/2225#issuecomment-880696737 - #wasm32-wasi, + wasm32-wasip1, sparc64-unknown-linux-gnu, wasm32-unknown-emscripten, x86_64-linux-android, diff --git a/build.rs b/build.rs index ead9cda6f011c..f9eb898cca9cf 100644 --- a/build.rs +++ b/build.rs @@ -17,6 +17,7 @@ const ALLOWED_CFGS: &'static [&'static str] = &[ "libc_const_extern_fn", "libc_const_extern_fn_unstable", "libc_deny_warnings", + "libc_ctest", ]; // Extra values to allow for check-cfg. diff --git a/ci/docker/wasm32-wasi/Dockerfile b/ci/docker/wasm32-wasi/Dockerfile deleted file mode 100644 index 80ae09f75038b..0000000000000 --- a/ci/docker/wasm32-wasi/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM ubuntu:23.10 - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - clang \ - curl \ - git \ - libc6-dev \ - make \ - xz-utils - -# Note that we're using `git reset --hard` to pin to a specific commit for -# verification for now. The sysroot is currently in somewhat of a state of flux -# and is expected to have breaking changes, so this is an attempt to mitigate -# those breaking changes on `libc`'s own CI -RUN git clone https://github.com/WebAssembly/wasi-libc && \ - cd wasi-libc && \ - git reset --hard ad5133410f66b93a2381db5b542aad5e0964db96 -RUN apt-get install -y --no-install-recommends llvm -RUN make -C wasi-libc install -j $(nproc) INSTALL_DIR=/wasi-libc - -RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/dev/wasmtime-dev-x86_64-linux.tar.xz | \ - tar xJf - -ENV PATH=$PATH:/wasmtime-dev-x86_64-linux -COPY docker/wasm32-wasi/clang.sh /wasi-libc/bin/clang - -RUN apt-get install -y --no-install-recommends lld -RUN ln -s /usr/bin/wasm-ld-10 /usr/bin/wasm-ld -ENV PATH=$PATH:/usr/lib/llvm-10/bin - -# Of note here is our clang wrapper which just executes a normal clang -# executable with the right sysroot, and then we're sure to turn off the -# crt-static feature to ensure that the CRT that we're specifying with `clang` -# is used. -ENV CARGO_TARGET_WASM32_WASI_RUNNER=wasmtime \ - CARGO_TARGET_WASM32_WASI_LINKER=/wasi-libc/bin/clang \ - CC_wasm32_wasi=/wasi-libc/bin/clang \ - PATH=$PATH:/rust/bin \ - RUSTFLAGS=-Ctarget-feature=-crt-static diff --git a/ci/docker/wasm32-wasi/clang.sh b/ci/docker/wasm32-wasi/clang.sh deleted file mode 100755 index 83f5da5ad6d81..0000000000000 --- a/ci/docker/wasm32-wasi/clang.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -exec /usr/bin/clang --target=wasm32-wasi --sysroot /wasi-libc/sysroot "$@" diff --git a/ci/docker/wasm32-wasip1/Dockerfile b/ci/docker/wasm32-wasip1/Dockerfile new file mode 100644 index 0000000000000..9ffb85671e3d3 --- /dev/null +++ b/ci/docker/wasm32-wasip1/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:24.04 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + clang \ + xz-utils + +# Wasmtime is used to execute tests and wasi-sdk is used to compile tests. +# Download appropriate versions here and configure various flags below. +ENV WASMTIME 24.0.0 +ENV WASI_SDK 24 + +RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v$WASMTIME/wasmtime-v$WASMTIME-x86_64-linux.tar.xz | \ + tar xJf - +ENV PATH=$PATH:/wasmtime-v$WASMTIME-x86_64-linux + +RUN curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_SDK/wasi-sdk-$WASI_SDK.0-x86_64-linux.deb +RUN dpkg -i ./wasi-sdk-*.deb + +# Note that `-D_WASI_EMULATED_PROCESS_CLOCKS` is used to enable access to +# clock-related defines even though they're emulated. Also note that the usage +# of `-Ctarget-feature=-crt-static` here forces usage of the external wasi-libc +# installed via `wasi-sdk` instead of the version that comes with the standard +# library. +ENV CARGO_TARGET_WASM32_WASIP1_RUNNER=wasmtime \ + CARGO_TARGET_WASM32_WASIP1_LINKER=/opt/wasi-sdk/bin/clang \ + CARGO_TARGET_WASM32_WASIP1_RUSTFLAGS="-lwasi-emulated-process-clocks -Ctarget-feature=-crt-static" \ + CC_wasm32_wasip1=/opt/wasi-sdk/bin/clang \ + CFLAGS_wasm32_wasip1=-D_WASI_EMULATED_PROCESS_CLOCKS \ + PATH=$PATH:/rust/bin diff --git a/libc-test/build.rs b/libc-test/build.rs index 26b1c550de22c..ec55fadea1ba9 100644 --- a/libc-test/build.rs +++ b/libc-test/build.rs @@ -1437,6 +1437,7 @@ fn test_wasi(target: &str) { "dirent.h", "errno.h", "fcntl.h", + "langinfo.h", "limits.h", "locale.h", "malloc.h", @@ -1448,6 +1449,7 @@ fn test_wasi(target: &str) { "stdio.h", "stdlib.h", "string.h", + "sys/ioctl.h", "sys/resource.h", "sys/select.h", "sys/socket.h", @@ -1456,16 +1458,20 @@ fn test_wasi(target: &str) { "sys/types.h", "sys/uio.h", "sys/utsname.h", - "sys/ioctl.h", "time.h", "unistd.h", "wasi/api.h", - "wasi/libc.h", "wasi/libc-find-relpath.h", "wasi/libc-nocwd.h", + "wasi/libc.h", "wchar.h", } + // Currently `ctest2` doesn't support macros-in-static-expressions and will + // panic on them. That affects `CLOCK_*` defines in wasi to set this here + // to omit them. + cfg.cfg("libc_ctest", None); + cfg.type_name(move |ty, is_struct, is_union| match ty { "FILE" | "fd_set" | "DIR" => ty.to_string(), t if is_union => format!("union {}", t), @@ -1484,20 +1490,27 @@ fn test_wasi(target: &str) { } }); - // Looks like LLD doesn't merge duplicate imports, so if the Rust - // code imports from a module and the C code also imports from a - // module we end up with two imports of function pointers which - // import the same thing but have different function pointers - cfg.skip_fn_ptrcheck(|f| f.starts_with("__wasi")); + // These have a different and internal type in header files and are only + // used here to generate a pointer to them in bindings so skip these tests. + cfg.skip_static(|c| c.starts_with("_CLOCK_")); + + cfg.skip_fn(|f| match f { + // This function doesn't actually exist in libc's header files + "__errno_location" => true, + + // The `timeout` argument to this function is `*const` in Rust but + // mutable in C which causes a mismatch. Avoiding breakage by changing + // this in wasi-libc and instead accepting that this is slightly + // different. + "select" => true, + + _ => false, + }); // d_name is declared as a flexible array in WASI libc, so it // doesn't support sizeof. cfg.skip_field(|s, field| s == "dirent" && field == "d_name"); - // Currently Rust/clang disagree on function argument ABI, so skip these - // tests. For more info see WebAssembly/tool-conventions#88 - cfg.skip_roundtrip(|_| true); - cfg.generate("../src/lib.rs", "main.rs"); } diff --git a/src/wasi.rs b/src/wasi.rs index f2823cd582f0e..4e894d2d1f403 100644 --- a/src/wasi.rs +++ b/src/wasi.rs @@ -248,14 +248,14 @@ pub const AT_SYMLINK_FOLLOW: c_int = 0x2; pub const AT_REMOVEDIR: c_int = 0x4; pub const UTIME_OMIT: c_long = 0xfffffffe; pub const UTIME_NOW: c_long = 0xffffffff; -pub const S_IFIFO: mode_t = 0o14_0000; +pub const S_IFIFO: mode_t = 0o1_0000; pub const S_IFCHR: mode_t = 0o2_0000; pub const S_IFBLK: mode_t = 0o6_0000; pub const S_IFDIR: mode_t = 0o4_0000; pub const S_IFREG: mode_t = 0o10_0000; pub const S_IFLNK: mode_t = 0o12_0000; pub const S_IFSOCK: mode_t = 0o14_0000; -pub const S_IFMT: mode_t = 0o16_0000; +pub const S_IFMT: mode_t = 0o17_0000; pub const S_IRWXO: mode_t = 0o0007; pub const S_IXOTH: mode_t = 0o0001; pub const S_IWOTH: mode_t = 0o0002; @@ -375,17 +375,26 @@ pub const _SC_PAGE_SIZE: ::c_int = _SC_PAGESIZE; pub const _SC_IOV_MAX: c_int = 60; pub const _SC_SYMLOOP_MAX: c_int = 173; -// unsafe code here is required in the stable, but not in nightly -#[allow(unused_unsafe)] -pub static CLOCK_MONOTONIC: clockid_t = unsafe { clockid_t(ptr_addr_of!(_CLOCK_MONOTONIC)) }; -#[allow(unused_unsafe)] -pub static CLOCK_PROCESS_CPUTIME_ID: clockid_t = - unsafe { clockid_t(ptr_addr_of!(_CLOCK_PROCESS_CPUTIME_ID)) }; -#[allow(unused_unsafe)] -pub static CLOCK_REALTIME: clockid_t = unsafe { clockid_t(ptr_addr_of!(_CLOCK_REALTIME)) }; -#[allow(unused_unsafe)] -pub static CLOCK_THREAD_CPUTIME_ID: clockid_t = - unsafe { clockid_t(ptr_addr_of!(_CLOCK_THREAD_CPUTIME_ID)) }; +cfg_if! { + if #[cfg(libc_ctest)] { + // skip these constants when this is active because `ctest` currently + // panics on parsing the constants below + } else { + // unsafe code here is required in the stable, but not in nightly + #[allow(unused_unsafe)] + pub static CLOCK_MONOTONIC: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_MONOTONIC)) }; + #[allow(unused_unsafe)] + pub static CLOCK_PROCESS_CPUTIME_ID: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_PROCESS_CPUTIME_ID)) }; + #[allow(unused_unsafe)] + pub static CLOCK_REALTIME: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_REALTIME)) }; + #[allow(unused_unsafe)] + pub static CLOCK_THREAD_CPUTIME_ID: clockid_t = + unsafe { clockid_t(ptr_addr_of!(_CLOCK_THREAD_CPUTIME_ID)) }; + } +} pub const ABDAY_1: ::nl_item = 0x20000; pub const ABDAY_2: ::nl_item = 0x20001;