Merge pull request #123 from kaeluka/failing-tests

Make failing tests possible, improve test output.

src/tests/encore/basic/Makefile

@@ -1,26 +1,12 @@
-ROOT_PATH=../../../..
-ENCOREC=$(ROOT_PATH)/release/encorec
-
-OUTFILES=$(shell ls *.out)
-
 ENCORE_SOURCES=$(shell ls *.enc)
 ENCORE_TARGETS=$(ENCORE_SOURCES:.enc=)
 
 test: embed_tl.o
-	@$(foreach PROG,$(ENCORE_TARGETS), make -Bqk $(PROG) || make -Bk $(PROG) || true;) 
-	@echo 
-	@echo Basic tests:
 	@./test.sh $(ENCORE_TARGETS)
 
 embed_tl.o: embed_tl.c embed_tl.h
 	clang -c embed_tl.c
 
-%: %.enc
-#ls -l
-	@rm -f $@
-	@echo "compiling '$@'"
-	@$(ENCOREC) $@.enc -clang -c
-
 clean:
 	rm -rf *.dSYM *_src $(ENCORE_TARGETS)
 	rm -f embed_tl.o

src/tests/encore/basic/test.sh

@@ -1,9 +1,43 @@
 #!/bin/bash
 
 function run_test {
+    # the function looks for either a file called "testname.out" or a
+    # script called "testname.chk".
+    #
+    # In the case of an .out file, the output of the executable has to
+    # match exactly; in the case of a .chk file, the output of
+    # compiling and (if successful) running the executable will be
+    # piped into that script. The test is successful iff the script
+    # returns normally (exit 0).
+
     program=$1
+    source=$1.enc
+    # compile and, if successful, run the program:
+    output=$(encorec $source && ./$program)
+    checking_script=./$program.chk
     expected=$program.out
-    ./$program | cmp $expected
+    if [ -x "$checking_script" ] ; then
+        if [ -e $expected ] ; then
+            echo "ERROR: both $checking_script and $expected exist, don't know which to use"
+            false
+        else
+            echo "$output" | ./$checking_script ||
+                (echo "ERROR: test $program's checking script failed.";
+                 false)
+        fi
+    else
+        if [ -e "$expected" ]; then
+            ./$program | cmp $expected ||
+                (echo "ERROR: test $program failed with output:";
+                 echo "vvvvvvvvvvvvvvvvvvvv"
+                 echo "$output"
+                 echo "^^^^^^^^^^^^^^^^^^^^";
+                 false)
+        else
+            echo "ERROR: incomplete test <$1>: neither checking script $checking_script, nor output file $expected is available"
+            exit 1
+        fi
+    fi
 }
 
 passed=0

src/tests/encore/basic/testutils.sh

@@ -0,0 +1,19 @@
+STDIN=$(cat)
+
+function FAIL {
+    MSG=$1
+    echo "ERROR: $1"
+    exit 1
+}
+
+# Requires that an error was reported in a specific line
+function error_at {
+    LINE=$1
+    (echo "$STDIN" | grep -A1 "(line $LINE, column" > /dev/null) || FAIL "expected error on line $LINE"
+}
+
+# Requires that an error was reported with a specific message
+function error_msg {
+    MSG=$1
+    (echo "$STDIN" | grep "$MSG" > /dev/null) || FAIL "expected error with message '$MSG'"
+}