From 9b4c5b034d5469b1da1dc3108bf814a43183d75c Mon Sep 17 00:00:00 2001 From: Stephan Brandauer Date: Thu, 16 Apr 2015 00:53:32 +0200 Subject: [PATCH] Make failing tests possible, improve test output. Until now it has been possible to test only that something succeeds (specifically, that the output matches exactly). This limited the tests to cases where we had determinism and where the expected output was exactly known. This commit extends test.sh to maintain the old functionality, but now ALSO allows checking scripts to be used instead of the old `.out` files. A checking script is an executable whose name is that of a test, but with the ending `.chk` -- if the test source if called `foo.enc`, the checking script is called `foo.chk` and it is executable. The file testutils.sh contains a few functions to make it easier to write tests and to make maintenance of tests easier, should error output change (then only this file has to be updated). The `make test` command will now compile `foo.enc` and try to run it (if compilation succeeds). The output of both compilation and execution will be piped into the checking script. The checking script can analyse the input and return successfully to signal that everything is ok, or with a failure code to signal an error. An example: File fail.enc: class Main def main() : void repeat i <- 100 { print x -- error should occur here } File fail.chk: #!/usr/bin/env bash source testutils.sh error_at 4 error_msg "Unbound variable 'x'" --- src/tests/encore/basic/Makefile | 14 ----------- src/tests/encore/basic/test.sh | 36 ++++++++++++++++++++++++++++- src/tests/encore/basic/testutils.sh | 19 +++++++++++++++ 3 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 src/tests/encore/basic/testutils.sh diff --git a/src/tests/encore/basic/Makefile b/src/tests/encore/basic/Makefile index 2ed00a396..6f08d97b3 100644 --- a/src/tests/encore/basic/Makefile +++ b/src/tests/encore/basic/Makefile @@ -1,26 +1,12 @@ -ROOT_PATH=../../../.. -ENCOREC=$(ROOT_PATH)/release/encorec - -OUTFILES=$(shell ls *.out) - ENCORE_SOURCES=$(shell ls *.enc) ENCORE_TARGETS=$(ENCORE_SOURCES:.enc=) test: embed_tl.o - @$(foreach PROG,$(ENCORE_TARGETS), make -Bqk $(PROG) || make -Bk $(PROG) || true;) - @echo - @echo Basic tests: @./test.sh $(ENCORE_TARGETS) embed_tl.o: embed_tl.c embed_tl.h clang -c embed_tl.c -%: %.enc -#ls -l - @rm -f $@ - @echo "compiling '$@'" - @$(ENCOREC) $@.enc -clang -c - clean: rm -rf *.dSYM *_src $(ENCORE_TARGETS) rm -f embed_tl.o diff --git a/src/tests/encore/basic/test.sh b/src/tests/encore/basic/test.sh index ee9aa7a04..3a66a07ed 100755 --- a/src/tests/encore/basic/test.sh +++ b/src/tests/encore/basic/test.sh @@ -1,9 +1,43 @@ #!/bin/bash function run_test { + # the function looks for either a file called "testname.out" or a + # script called "testname.chk". + # + # In the case of an .out file, the output of the executable has to + # match exactly; in the case of a .chk file, the output of + # compiling and (if successful) running the executable will be + # piped into that script. The test is successful iff the script + # returns normally (exit 0). + program=$1 + source=$1.enc + # compile and, if successful, run the program: + output=$(encorec $source && ./$program) + checking_script=./$program.chk expected=$program.out - ./$program | cmp $expected + if [ -x "$checking_script" ] ; then + if [ -e $expected ] ; then + echo "ERROR: both $checking_script and $expected exist, don't know which to use" + false + else + echo "$output" | ./$checking_script || + (echo "ERROR: test $program's checking script failed."; + false) + fi + else + if [ -e "$expected" ]; then + ./$program | cmp $expected || + (echo "ERROR: test $program failed with output:"; + echo "vvvvvvvvvvvvvvvvvvvv" + echo "$output" + echo "^^^^^^^^^^^^^^^^^^^^"; + false) + else + echo "ERROR: incomplete test <$1>: neither checking script $checking_script, nor output file $expected is available" + exit 1 + fi + fi } passed=0 diff --git a/src/tests/encore/basic/testutils.sh b/src/tests/encore/basic/testutils.sh new file mode 100644 index 000000000..000147b3f --- /dev/null +++ b/src/tests/encore/basic/testutils.sh @@ -0,0 +1,19 @@ +STDIN=$(cat) + +function FAIL { + MSG=$1 + echo "ERROR: $1" + exit 1 +} + +# Requires that an error was reported in a specific line +function error_at { + LINE=$1 + (echo "$STDIN" | grep -A1 "(line $LINE, column" > /dev/null) || FAIL "expected error on line $LINE" +} + +# Requires that an error was reported with a specific message +function error_msg { + MSG=$1 + (echo "$STDIN" | grep "$MSG" > /dev/null) || FAIL "expected error with message '$MSG'" +}