Skip to content

Commit 2b853ff

Browse files
abrownabrown
authored
test: improve test infrastructure (#554)
This change represents a rather large re-design in how `wasi-libc` builds and runs its tests. Initially, #346 retrieved the `libc-test` repository and built a subset of those tests to give us some amount of test coverage. Later, because there was no way to add custom C tests, #522 added a `smoke` directory which allowed this. But (a) each of these test suites was built and run separately and (b) it was unclear how to add more tests flexibly--some tests should only run on `*p2` targets or `*-threads` targets, e.g. This change reworks all of this so that all tests are built the same way, in the same place. For downloaded tests like those from `libc-test`, I chose to add "stub tests" that `#include` the original version. This not only keeps all enabled tests in one place, it also allows us to add "directives," C comments that the `Makefile` uses to filter out tests for certain targets or add special compile, link or run flags. These rudimentary scripts, along with other Bash logic I moved out of the Makefile now live in the `scripts` directory. Finally, all of this is explained more clearly in an updated `README.md`. The hope with documenting this a bit better is that it would be easier for drive-by contributors to be able to either dump in new C tests for regressions they may find or enable more libc-tests. As of my current count, we only enable 40/75 of libc-test's functional tests, 0/228 math tests, 0/69 regression tests, and 0/79 API tests. Though many of these may not apply to WASI programs, it would be nice to explore how many more of these tests can be enabled to increase wasi-libc's test coverage. This change should explain how to do that and, with directives, make it possible to condition how the tests compile and run.
1 parent 29c22a4 commit 2b853ff

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+529
-164
lines changed

‎.github/workflows/main.yml

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -125,17 +125,13 @@ jobs:
125125
export WASIP1_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip1/)
126126
export WASIP2_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip2/)
127127
mkdir -p $WASI_DIR $WASIP1_DIR $WASIP2_DIR
128-
cp download/libclang_rt.builtins-wasm32.a $WASI_DIR
129-
cp download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
130-
cp download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
128+
cp build/download/libclang_rt.builtins-wasm32.a $WASI_DIR
129+
cp build/download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
130+
cp build/download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
131131
TARGET_TRIPLE=wasm32-wasi make test
132-
rm -r build
133132
TARGET_TRIPLE=wasm32-wasip1 make test
134-
rm -r build
135133
TARGET_TRIPLE=wasm32-wasip2 make test
136-
rm -r build
137134
TARGET_TRIPLE=wasm32-wasi-threads make test
138-
rm -r build
139135
TARGET_TRIPLE=wasm32-wasip1-threads make test
140136
# The older version of Clang does not provide the expected symbol for the
141137
# test entrypoints: `undefined symbol: __main_argc_argv`.

‎test/.gitignore

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,2 @@
11
build
2-
download
32
run
4-
5-
smoke/*.dir

‎test/Makefile

Lines changed: 121 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -9,37 +9,43 @@
99
# - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
1010
# Wasmtime)
1111

12-
test: run run_smoke
13-
14-
# Unlike the `libc-test` directory, we will output all artifacts to the `build`
15-
# directory (keeping with the `wasi-libc` conventions).
16-
OBJDIR ?= $(CURDIR)/build
17-
DOWNDIR ?= $(CURDIR)/download
12+
test: run
1813

14+
# Decide which target to build for and which libc to use.
1915
TARGET_TRIPLE ?= wasm32-wasi
2016

17+
# Setup various paths used by the tests.
18+
OBJDIR ?= build/$(TARGET_TRIPLE)
19+
DOWNDIR ?= build/download
20+
SRCDIR ?= src
21+
RUNDIR ?= run/$(TARGET_TRIPLE)
22+
23+
# We also need to know the location the wasi-libc sysroot we're building
24+
# against.
25+
SYSROOT_DIR ?= ../sysroot
26+
SYSROOT := $(SYSROOT_DIR)/lib/$(TARGET_TRIPLE)
27+
$(SYSROOT):
28+
@echo "No sysroot for $(TARGET_TRIPLE) available at $(SYSROOT_DIR); to build it, e.g.:"
29+
@echo " cd $(dir $(SYSROOT_DIR))"
30+
@echo " make TARGET_TRIPLE=$(TARGET_TRIPLE)"
31+
@exit 1
32+
33+
2134
##### DOWNLOAD #################################################################
2235

2336
LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
2437
LIBC_TEST = $(DOWNDIR)/libc-test
2538
LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/libclang_rt.builtins-wasm32-wasi-24.0.tar.gz
2639
LIBRT = $(DOWNDIR)/libclang_rt.builtins-wasm32.a
2740
WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-linux.tar.xz
28-
WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
41+
WASMTIME = $(abspath $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime)
2942
WASM_TOOLS_URL ?= https://github.com/bytecodealliance/wasm-tools/releases/download/v1.220.0/wasm-tools-1.220.0-x86_64-linux.tar.gz
3043
WASM_TOOLS = $(DOWNDIR)/$(shell basename $(WASM_TOOLS_URL) .tar.gz)/wasm-tools
3144
ADAPTER_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasi_snapshot_preview1.command.wasm
3245
ADAPTER = $(DOWNDIR)/wasi_snapshot_preview1.command.wasm
3346

34-
TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
35-
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
36-
TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
37-
endif
38-
39-
download: $(TO_DOWNLOAD)
40-
4147
$(DOWNDIR):
42-
mkdir -p download
48+
@mkdir -p $@
4349

4450
$(LIBC_TEST): | $(DOWNDIR)
4551
git clone --depth 1 $(LIBC_TEST_URL) $@
@@ -61,151 +67,149 @@ $(WASM_TOOLS): | $(DOWNDIR)
6167
$(ADAPTER): | $(DOWNDIR)
6268
wget --no-clobber --directory-prefix=$(DOWNDIR) $(ADAPTER_URL)
6369

70+
# Target to download all necessary dependencies.
71+
TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
72+
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
73+
TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
74+
endif
75+
DOWNLOADED := $(DOWNDIR)/downloaded.stamp
76+
$(DOWNLOADED): $(TO_DOWNLOAD)
77+
touch $@
78+
download: $(DOWNLOADED)
79+
6480
clean::
6581
rm -rf $(DOWNDIR)
6682

67-
##### BUILD ####################################################################
83+
##### INFRA ####################################################################
84+
85+
INFRA_OBJDIR := $(OBJDIR)/common
86+
$(INFRA_OBJDIR):
87+
@mkdir -p $@
6888

69-
# For now, we list out the tests that we can currently build and run. This is
70-
# heavily focused on the functional tests; in the future it would be good to
71-
# fill out the missing tests and also include some `src/api` and `src/math`
72-
# tests (TODO).
73-
TESTS := \
74-
$(LIBC_TEST)/src/functional/argv.c \
75-
$(LIBC_TEST)/src/functional/basename.c \
76-
$(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
77-
$(LIBC_TEST)/src/functional/clock_gettime.c \
78-
$(LIBC_TEST)/src/functional/crypt.c \
79-
$(LIBC_TEST)/src/functional/dirname.c \
80-
$(LIBC_TEST)/src/functional/env.c \
81-
$(LIBC_TEST)/src/functional/fnmatch.c \
82-
$(LIBC_TEST)/src/functional/iconv_open.c \
83-
$(LIBC_TEST)/src/functional/mbc.c \
84-
$(LIBC_TEST)/src/functional/memstream.c \
85-
$(LIBC_TEST)/src/functional/qsort.c \
86-
$(LIBC_TEST)/src/functional/random.c \
87-
$(LIBC_TEST)/src/functional/search_hsearch.c \
88-
$(LIBC_TEST)/src/functional/search_insque.c \
89-
$(LIBC_TEST)/src/functional/search_lsearch.c \
90-
$(LIBC_TEST)/src/functional/search_tsearch.c \
91-
$(LIBC_TEST)/src/functional/snprintf.c \
92-
$(LIBC_TEST)/src/functional/sscanf.c \
93-
$(LIBC_TEST)/src/functional/strftime.c \
94-
$(LIBC_TEST)/src/functional/string.c \
95-
$(LIBC_TEST)/src/functional/string_memcpy.c \
96-
$(LIBC_TEST)/src/functional/string_memmem.c \
97-
$(LIBC_TEST)/src/functional/string_memset.c \
98-
$(LIBC_TEST)/src/functional/string_strchr.c \
99-
$(LIBC_TEST)/src/functional/string_strcspn.c \
100-
$(LIBC_TEST)/src/functional/string_strstr.c \
101-
$(LIBC_TEST)/src/functional/strtod.c \
102-
$(LIBC_TEST)/src/functional/strtod_long.c \
103-
$(LIBC_TEST)/src/functional/strtod_simple.c \
104-
$(LIBC_TEST)/src/functional/strtof.c \
105-
$(LIBC_TEST)/src/functional/strtol.c \
106-
$(LIBC_TEST)/src/functional/swprintf.c \
107-
$(LIBC_TEST)/src/functional/tgmath.c \
108-
$(LIBC_TEST)/src/functional/udiv.c \
109-
$(LIBC_TEST)/src/functional/wcsstr.c \
110-
$(LIBC_TEST)/src/functional/wcstol.c
111-
112-
# Part of the problem including more tests is that the `libc-test`
113-
# infrastructure code is not all Wasm-compilable. As we include more tests
114-
# above, this list will also likely need to grow.
115-
COMMON_TEST_INFRA = \
116-
$(LIBC_TEST)/src/common/path.c \
89+
# Build the common test infrastructure. Part of the problem including more tests
90+
# is that the `libc-test` infrastructure code is not all Wasm-compilable. As we
91+
# include more tests above, this list will also likely need to grow.
92+
INFRA_FILES = $(LIBC_TEST)/src/common/path.c \
11793
$(LIBC_TEST)/src/common/print.c \
11894
$(LIBC_TEST)/src/common/rand.c \
11995
$(LIBC_TEST)/src/common/utf8.c
96+
$(INFRA_FILES): $(DOWNLOADED)
97+
INFRA_WASM_OBJS := $(patsubst $(LIBC_TEST)/src/common/%.c,$(OBJDIR)/common/%.wasm.o,$(INFRA_FILES))
98+
$(OBJDIR)/common/%.wasm.o: $(LIBC_TEST)/src/common/%.c | $(INFRA_OBJDIR)
99+
$(CC) $(CFLAGS) -c $< -o $@
100+
101+
# Also, include the `libc-test` infrastructure headers.
102+
INFRA_HEADERS_DIR := $(LIBC_TEST)/src/common
103+
INFRA_HEADERS := $(shell find $(INFRA_HEADERS_DIR) -name '*.h')
104+
$(INFRA_HEADERS): $(DOWNLOADED)
105+
106+
##### BUILD ####################################################################
120107

121108
# Create various lists containing the various artifacts to be built: mainly,
122109
# $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
123110
# the $(WASMS) tests.
124-
NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
125-
WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.core.wasm)
126-
WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
127-
INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
111+
ALL_TESTS := $(shell find $(SRCDIR) -name '*.c')
112+
TESTS := $(shell TARGET_TRIPLE=$(TARGET_TRIPLE) scripts/filter.py $(ALL_TESTS))
113+
WASM_OBJS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.wasm.o)
128114
WASM_OBJS += $(INFRA_WASM_OBJS)
129-
DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
130-
OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
115+
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
116+
WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.component.wasm)
117+
else
118+
WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.core.wasm)
119+
endif
120+
131121

132-
# Allow $(CC) to be set from the command line; ?= doesn't work for CC because
133-
# make has a default value for it.
122+
# Setup the compiler. We allow $(CC) to be set from the command line; ?= doesn't
123+
# work for CC because make has a default value for it.
134124
ifeq ($(origin CC), default)
135125
CC := clang
136126
endif
137127
LDFLAGS ?=
138-
CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=../sysroot
128+
CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=$(SYSROOT_DIR)
139129
# Always include the `libc-test` infrastructure headers.
140-
CFLAGS += -I$(LIBC_TEST)/src/common
130+
CFLAGS += -I$(INFRA_HEADERS_DIR)
141131
ifneq ($(findstring -threads,$(TARGET_TRIPLE)),)
142132
CFLAGS += -pthread
143133
endif
144134

145-
# Compile each selected test using Clang. Note that failures here are likely
146-
# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
147-
# This location is system-dependent, but could be fixed by something like:
148-
# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
149-
# $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
150-
build: download $(WASMS)
135+
# Build up all the `*.wasm.o` object files; these are the same regardless of
136+
# whether we're building core modules or components.
137+
$(WASM_OBJS): $(INFRA_HEADERS)
138+
$(OBJDIR)/%.wasm.o: $(SRCDIR)/%.c $(DOWNLOADED) $(SYSROOT)
139+
@mkdir -p $(@D)
140+
$(CC) $(CFLAGS) $(shell scripts/add-flags.py CFLAGS $<) -c $< -o $@
151141

152-
$(WASMS): | $(OBJDIRS)
142+
# Build up all the `*.wasm` files.
143+
obj_to_c = $(patsubst $(OBJDIR)/%.wasm.o,$(SRCDIR)/%.c,$1)
153144
$(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
154-
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
145+
@mkdir -p $(@D)
146+
$(CC) $(CFLAGS) $(LDFLAGS) $(shell scripts/add-flags.py LDFLAGS $(call obj_to_c,$<)) $^ -o $@
155147

156-
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
157-
$(OBJDIR)/%.wasm: $(OBJDIR)/%.core.wasm
148+
# For wasip2, we include an additional step to wrap up the core module into
149+
# a component.
150+
$(OBJDIR)/%.component.wasm: $(OBJDIR)/%.core.wasm
158151
$(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@
159-
endif
160-
161-
$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
162-
$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
163-
$(CC) $(CFLAGS) -c -o $@ $<
164152

165-
$(OBJDIRS):
166-
mkdir -p $@
153+
# Compile each selected test using Clang. Note that failures here are likely
154+
# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
155+
# This location is system-dependent, but could be fixed by something like:
156+
# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
157+
# $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
158+
build: $(DOWNLOADED) $(WASMS)
167159

168160
clean::
169161
rm -rf $(OBJDIR)
170162

171-
##### RUN ######################################################################
163+
##### GENERATE #################################################################
172164

173-
ENGINE ?= $(WASMTIME) run
174-
ERRS:=$(WASMS:%.core.wasm=%.wasm.err)
165+
# Not all of the downloaded `libc-test` tests can be built and run with
166+
# `wasi-libc`. Thus, we only include the subset that can be in `src/libc-test`
167+
# as stub files that `#include` the original test files. When we want to add
168+
# more tests, though, the `generate-stubs` target will generate stubs for the
169+
# missing tests which we can delete or alter as needed.
175170

176-
# Use the provided Wasm engine to execute each test, emitting its output into
177-
# a `.err` file.
178-
run: build $(ERRS)
179-
@echo "Tests passed"
171+
STUBDIR := $(SRCDIR)/libc-test
172+
generate-stubs:
173+
FROM_DIR=$(LIBC_TEST) TO_DIR=$(STUBDIR) scripts/generate-stubs.sh
180174

181-
$(ERRS): | $(OBJDIRS)
175+
##### RUN ######################################################################
182176

177+
ENGINE ?= $(WASMTIME) run
183178
ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
184-
%.wasm.err: %.wasm
185-
$(ENGINE) --wasm component-model $< >$@
179+
ENGINE += --wasm component-model
180+
OBJPAT := $(OBJDIR)/%.component.wasm
186181
else
187-
%.wasm.err: %.core.wasm
188-
$(ENGINE) $< >$@
182+
OBJPAT := $(OBJDIR)/%.core.wasm
189183
endif
190184

191-
clean::
192-
rm -rf $(OBJDIR)/*/*.err
185+
# Each Wasm test is run every time, generating a folder containing a `cmd.sh`
186+
# script and an `output.log` file (see `scripts/run-test.sh` for details). The
187+
# `success` file is never generated, which means the test will rerun every time.
188+
# To ignore a test temporarily, `touch .../success:`.
189+
RUNTESTS:=$(WASMS:$(OBJPAT)=$(RUNDIR)/%/success)
190+
wasm_to_c = $(patsubst $(OBJPAT),$(SRCDIR)/%.c,$1)
191+
$(RUNDIR)/%/success: $(OBJPAT)
192+
@mkdir -p $(@D)
193+
@DIR="$(abspath $(@D))" \
194+
WASM="$(abspath $<)" \
195+
ENGINE="$(ENGINE) $(shell scripts/add-flags.py RUN $(call wasm_to_c,$<))" \
196+
scripts/run-test.sh
193197

194-
##### SMOKE TEST SUITE #########################################################
198+
# Use the provided Wasm engine to execute each test, emitting its output into
199+
# a `.err` file.
200+
run: build $(RUNTESTS)
201+
@if scripts/failed-tests.sh $(RUNDIR); then \
202+
echo "Tests passed"; \
203+
else \
204+
echo "Tests failed:"; \
205+
VERBOSE=1 scripts/failed-tests.sh $(RUNDIR); \
206+
fi
195207

196-
include smoke/smoke.mk
208+
clean::
209+
rm -rf $(RUNDIR)
197210

198211
##### MISC #####################################################################
199212

200213
# Note: the `clean` target has been built up by all of the previous sections.
201214

202-
debug:
203-
@echo NAMES $(NAMES)
204-
@echo TESTS $(TESTS)
205-
@echo WASMS $(WASMS)
206-
@echo WASM_OBJS $(WASM_OBJS)
207-
@echo ERRS $(ERRS)
208-
@echo DIRS $(DIRS)
209-
@echo OBJDIRS $(OBJDIRS)
210-
211-
.PHONY: test download build run clean
215+
.PHONY: test download build run generate-stubs clean

0 commit comments

Comments
 (0)