Skip to content

Commit

Permalink
test: improve test infrastructure (#554)
Browse files Browse the repository at this point in the history 
This change represents a rather large re-design in how `wasi-libc`
builds and runs its tests. Initially, #346 retrieved the `libc-test`
repository and built a subset of those tests to give us some amount of
test coverage. Later, because there was no way to add custom C tests,
#522 added a `smoke` directory which allowed this. But (a) each of these
test suites was built and run separately and (b) it was unclear how to
add more tests flexibly--some tests should only run on `*p2` targets or
`*-threads` targets, e.g.

This change reworks all of this so that all tests are built the same
way, in the same place. For downloaded tests like those from
`libc-test`, I chose to add "stub tests" that `#include` the original
version. This not only keeps all enabled tests in one place, it also
allows us to add "directives," C comments that the `Makefile` uses to
filter out tests for certain targets or add special compile, link or run
flags. These rudimentary scripts, along with other Bash logic I moved
out of the Makefile now live in the `scripts` directory.

Finally, all of this is explained more clearly in an updated
`README.md`. The hope with documenting this a bit better is that it
would be easier for drive-by contributors to be able to either dump in
new C tests for regressions they may find or enable more libc-tests. As
of my current count, we only enable 40/75 of libc-test's functional
tests, 0/228 math tests, 0/69 regression tests, and 0/79 API tests.
Though many of these may not apply to WASI programs, it would be nice to
explore how many more of these tests can be enabled to increase
wasi-libc's test coverage. This change should explain how to do that
and, with directives, make it possible to condition how the tests
compile and run.
  • Loading branch information
@abrown
abrown authored Dec 9, 2024
1 parent 29c22a4 commit 2b853ff
Show file tree
Hide file tree
Showing 49 changed files with 529 additions and 164 deletions.
10 changes: 3 additions & 7 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,17 +125,13 @@ jobs:
export WASIP1_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip1/)
export WASIP2_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip2/)
mkdir -p $WASI_DIR $WASIP1_DIR $WASIP2_DIR
cp download/libclang_rt.builtins-wasm32.a $WASI_DIR
cp download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
cp download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
cp build/download/libclang_rt.builtins-wasm32.a $WASI_DIR
cp build/download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
cp build/download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
TARGET_TRIPLE=wasm32-wasi make test
rm -r build
TARGET_TRIPLE=wasm32-wasip1 make test
rm -r build
TARGET_TRIPLE=wasm32-wasip2 make test
rm -r build
TARGET_TRIPLE=wasm32-wasi-threads make test
rm -r build
TARGET_TRIPLE=wasm32-wasip1-threads make test
# The older version of Clang does not provide the expected symbol for the
# test entrypoints: `undefined symbol: __main_argc_argv`.
Expand Down
3 changes: 0 additions & 3 deletions test/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,2 @@
build
download
run

smoke/*.dir
238 changes: 121 additions & 117 deletions test/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,37 +9,43 @@
# - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
# Wasmtime)

test: run run_smoke

# Unlike the `libc-test` directory, we will output all artifacts to the `build`
# directory (keeping with the `wasi-libc` conventions).
OBJDIR ?= $(CURDIR)/build
DOWNDIR ?= $(CURDIR)/download
test: run

# Decide which target to build for and which libc to use.
TARGET_TRIPLE ?= wasm32-wasi

# Setup various paths used by the tests.
OBJDIR ?= build/$(TARGET_TRIPLE)
DOWNDIR ?= build/download
SRCDIR ?= src
RUNDIR ?= run/$(TARGET_TRIPLE)

# We also need to know the location the wasi-libc sysroot we're building
# against.
SYSROOT_DIR ?= ../sysroot
SYSROOT := $(SYSROOT_DIR)/lib/$(TARGET_TRIPLE)
$(SYSROOT):
@echo "No sysroot for $(TARGET_TRIPLE) available at $(SYSROOT_DIR); to build it, e.g.:"
@echo " cd $(dir $(SYSROOT_DIR))"
@echo " make TARGET_TRIPLE=$(TARGET_TRIPLE)"
@exit 1


##### DOWNLOAD #################################################################

LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
LIBC_TEST = $(DOWNDIR)/libc-test
LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/libclang_rt.builtins-wasm32-wasi-24.0.tar.gz
LIBRT = $(DOWNDIR)/libclang_rt.builtins-wasm32.a
WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-linux.tar.xz
WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
WASMTIME = $(abspath $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime)
WASM_TOOLS_URL ?= https://github.com/bytecodealliance/wasm-tools/releases/download/v1.220.0/wasm-tools-1.220.0-x86_64-linux.tar.gz
WASM_TOOLS = $(DOWNDIR)/$(shell basename $(WASM_TOOLS_URL) .tar.gz)/wasm-tools
ADAPTER_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasi_snapshot_preview1.command.wasm
ADAPTER = $(DOWNDIR)/wasi_snapshot_preview1.command.wasm

TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
endif

download: $(TO_DOWNLOAD)

$(DOWNDIR):
mkdir -p download
@mkdir -p $@

$(LIBC_TEST): | $(DOWNDIR)
git clone --depth 1 $(LIBC_TEST_URL) $@
Expand All @@ -61,151 +67,149 @@ $(WASM_TOOLS): | $(DOWNDIR)
$(ADAPTER): | $(DOWNDIR)
wget --no-clobber --directory-prefix=$(DOWNDIR) $(ADAPTER_URL)

# Target to download all necessary dependencies.
TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
endif
DOWNLOADED := $(DOWNDIR)/downloaded.stamp
$(DOWNLOADED): $(TO_DOWNLOAD)
touch $@
download: $(DOWNLOADED)

clean::
rm -rf $(DOWNDIR)

##### BUILD ####################################################################
##### INFRA ####################################################################

INFRA_OBJDIR := $(OBJDIR)/common
$(INFRA_OBJDIR):
@mkdir -p $@

# For now, we list out the tests that we can currently build and run. This is
# heavily focused on the functional tests; in the future it would be good to
# fill out the missing tests and also include some `src/api` and `src/math`
# tests (TODO).
TESTS := \
$(LIBC_TEST)/src/functional/argv.c \
$(LIBC_TEST)/src/functional/basename.c \
$(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
$(LIBC_TEST)/src/functional/clock_gettime.c \
$(LIBC_TEST)/src/functional/crypt.c \
$(LIBC_TEST)/src/functional/dirname.c \
$(LIBC_TEST)/src/functional/env.c \
$(LIBC_TEST)/src/functional/fnmatch.c \
$(LIBC_TEST)/src/functional/iconv_open.c \
$(LIBC_TEST)/src/functional/mbc.c \
$(LIBC_TEST)/src/functional/memstream.c \
$(LIBC_TEST)/src/functional/qsort.c \
$(LIBC_TEST)/src/functional/random.c \
$(LIBC_TEST)/src/functional/search_hsearch.c \
$(LIBC_TEST)/src/functional/search_insque.c \
$(LIBC_TEST)/src/functional/search_lsearch.c \
$(LIBC_TEST)/src/functional/search_tsearch.c \
$(LIBC_TEST)/src/functional/snprintf.c \
$(LIBC_TEST)/src/functional/sscanf.c \
$(LIBC_TEST)/src/functional/strftime.c \
$(LIBC_TEST)/src/functional/string.c \
$(LIBC_TEST)/src/functional/string_memcpy.c \
$(LIBC_TEST)/src/functional/string_memmem.c \
$(LIBC_TEST)/src/functional/string_memset.c \
$(LIBC_TEST)/src/functional/string_strchr.c \
$(LIBC_TEST)/src/functional/string_strcspn.c \
$(LIBC_TEST)/src/functional/string_strstr.c \
$(LIBC_TEST)/src/functional/strtod.c \
$(LIBC_TEST)/src/functional/strtod_long.c \
$(LIBC_TEST)/src/functional/strtod_simple.c \
$(LIBC_TEST)/src/functional/strtof.c \
$(LIBC_TEST)/src/functional/strtol.c \
$(LIBC_TEST)/src/functional/swprintf.c \
$(LIBC_TEST)/src/functional/tgmath.c \
$(LIBC_TEST)/src/functional/udiv.c \
$(LIBC_TEST)/src/functional/wcsstr.c \
$(LIBC_TEST)/src/functional/wcstol.c

# Part of the problem including more tests is that the `libc-test`
# infrastructure code is not all Wasm-compilable. As we include more tests
# above, this list will also likely need to grow.
COMMON_TEST_INFRA = \
$(LIBC_TEST)/src/common/path.c \
# Build the common test infrastructure. Part of the problem including more tests
# is that the `libc-test` infrastructure code is not all Wasm-compilable. As we
# include more tests above, this list will also likely need to grow.
INFRA_FILES = $(LIBC_TEST)/src/common/path.c \
$(LIBC_TEST)/src/common/print.c \
$(LIBC_TEST)/src/common/rand.c \
$(LIBC_TEST)/src/common/utf8.c
$(INFRA_FILES): $(DOWNLOADED)
INFRA_WASM_OBJS := $(patsubst $(LIBC_TEST)/src/common/%.c,$(OBJDIR)/common/%.wasm.o,$(INFRA_FILES))
$(OBJDIR)/common/%.wasm.o: $(LIBC_TEST)/src/common/%.c | $(INFRA_OBJDIR)
$(CC) $(CFLAGS) -c $< -o $@

# Also, include the `libc-test` infrastructure headers.
INFRA_HEADERS_DIR := $(LIBC_TEST)/src/common
INFRA_HEADERS := $(shell find $(INFRA_HEADERS_DIR) -name '*.h')
$(INFRA_HEADERS): $(DOWNLOADED)

##### BUILD ####################################################################

# Create various lists containing the various artifacts to be built: mainly,
# $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
# the $(WASMS) tests.
NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.core.wasm)
WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
ALL_TESTS := $(shell find $(SRCDIR) -name '*.c')
TESTS := $(shell TARGET_TRIPLE=$(TARGET_TRIPLE) scripts/filter.py $(ALL_TESTS))
WASM_OBJS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.wasm.o)
WASM_OBJS += $(INFRA_WASM_OBJS)
DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.component.wasm)
else
WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.core.wasm)
endif


# Allow $(CC) to be set from the command line; ?= doesn't work for CC because
# make has a default value for it.
# Setup the compiler. We allow $(CC) to be set from the command line; ?= doesn't
# work for CC because make has a default value for it.
ifeq ($(origin CC), default)
CC := clang
endif
LDFLAGS ?=
CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=../sysroot
CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=$(SYSROOT_DIR)
# Always include the `libc-test` infrastructure headers.
CFLAGS += -I$(LIBC_TEST)/src/common
CFLAGS += -I$(INFRA_HEADERS_DIR)
ifneq ($(findstring -threads,$(TARGET_TRIPLE)),)
CFLAGS += -pthread
endif

# Compile each selected test using Clang. Note that failures here are likely
# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
# This location is system-dependent, but could be fixed by something like:
# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
# $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
build: download $(WASMS)
# Build up all the `*.wasm.o` object files; these are the same regardless of
# whether we're building core modules or components.
$(WASM_OBJS): $(INFRA_HEADERS)
$(OBJDIR)/%.wasm.o: $(SRCDIR)/%.c $(DOWNLOADED) $(SYSROOT)
@mkdir -p $(@D)
$(CC) $(CFLAGS) $(shell scripts/add-flags.py CFLAGS $<) -c $< -o $@

$(WASMS): | $(OBJDIRS)
# Build up all the `*.wasm` files.
obj_to_c = $(patsubst $(OBJDIR)/%.wasm.o,$(SRCDIR)/%.c,$1)
$(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
@mkdir -p $(@D)
$(CC) $(CFLAGS) $(LDFLAGS) $(shell scripts/add-flags.py LDFLAGS $(call obj_to_c,$<)) $^ -o $@

ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
$(OBJDIR)/%.wasm: $(OBJDIR)/%.core.wasm
# For wasip2, we include an additional step to wrap up the core module into
# a component.
$(OBJDIR)/%.component.wasm: $(OBJDIR)/%.core.wasm
$(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@
endif

$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
$(CC) $(CFLAGS) -c -o $@ $<

$(OBJDIRS):
mkdir -p $@
# Compile each selected test using Clang. Note that failures here are likely
# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
# This location is system-dependent, but could be fixed by something like:
# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
# $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
build: $(DOWNLOADED) $(WASMS)

clean::
rm -rf $(OBJDIR)

##### RUN ######################################################################
##### GENERATE #################################################################

ENGINE ?= $(WASMTIME) run
ERRS:=$(WASMS:%.core.wasm=%.wasm.err)
# Not all of the downloaded `libc-test` tests can be built and run with
# `wasi-libc`. Thus, we only include the subset that can be in `src/libc-test`
# as stub files that `#include` the original test files. When we want to add
# more tests, though, the `generate-stubs` target will generate stubs for the
# missing tests which we can delete or alter as needed.

# Use the provided Wasm engine to execute each test, emitting its output into
# a `.err` file.
run: build $(ERRS)
@echo "Tests passed"
STUBDIR := $(SRCDIR)/libc-test
generate-stubs:
FROM_DIR=$(LIBC_TEST) TO_DIR=$(STUBDIR) scripts/generate-stubs.sh

$(ERRS): | $(OBJDIRS)
##### RUN ######################################################################

ENGINE ?= $(WASMTIME) run
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
%.wasm.err: %.wasm
$(ENGINE) --wasm component-model $< >$@
ENGINE += --wasm component-model
OBJPAT := $(OBJDIR)/%.component.wasm
else
%.wasm.err: %.core.wasm
$(ENGINE) $< >$@
OBJPAT := $(OBJDIR)/%.core.wasm
endif

clean::
rm -rf $(OBJDIR)/*/*.err
# Each Wasm test is run every time, generating a folder containing a `cmd.sh`
# script and an `output.log` file (see `scripts/run-test.sh` for details). The
# `success` file is never generated, which means the test will rerun every time.
# To ignore a test temporarily, `touch .../success:`.
RUNTESTS:=$(WASMS:$(OBJPAT)=$(RUNDIR)/%/success)
wasm_to_c = $(patsubst $(OBJPAT),$(SRCDIR)/%.c,$1)
$(RUNDIR)/%/success: $(OBJPAT)
@mkdir -p $(@D)
@DIR="$(abspath $(@D))" \
WASM="$(abspath $<)" \
ENGINE="$(ENGINE) $(shell scripts/add-flags.py RUN $(call wasm_to_c,$<))" \
scripts/run-test.sh

##### SMOKE TEST SUITE #########################################################
# Use the provided Wasm engine to execute each test, emitting its output into
# a `.err` file.
run: build $(RUNTESTS)
@if scripts/failed-tests.sh $(RUNDIR); then \
echo "Tests passed"; \
else \
echo "Tests failed:"; \
VERBOSE=1 scripts/failed-tests.sh $(RUNDIR); \
fi

include smoke/smoke.mk
clean::
rm -rf $(RUNDIR)

##### MISC #####################################################################

# Note: the `clean` target has been built up by all of the previous sections.

debug:
@echo NAMES $(NAMES)
@echo TESTS $(TESTS)
@echo WASMS $(WASMS)
@echo WASM_OBJS $(WASM_OBJS)
@echo ERRS $(ERRS)
@echo DIRS $(DIRS)
@echo OBJDIRS $(OBJDIRS)

.PHONY: test download build run clean
.PHONY: test download build run generate-stubs clean
Loading

0 comments on commit 2b853ff

Please sign in to comment.