Merge pull request #6465 from layus/fix-test-crashes

tests: Distinguish crashes from expected failures
NixOS · May 3, 2022 · cae7473 · cae7473
2 parents 61289ce + 275f856
commit cae7473
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 10 deletions.
diff --git a/tests/common.sh.in b/tests/common.sh.in
@@ -157,11 +157,12 @@ expect() {
     local expected res
     expected="$1"
     shift
-    set +e
-    "$@"
-    res="$?"
-    set -e
-    [[ $res -eq $expected ]]
+    "$@" || res="$?"
+    if [[ $res -ne $expected ]]; then
+        echo "Expected '$expected' but got '$res' while running '$*'"
+        return 1
+    fi
+    return 0
 }
 
 needLocalStore() {

diff --git a/tests/lang.sh b/tests/lang.sh
@@ -4,6 +4,7 @@ export TEST_VAR=foo # for eval-okay-getenv.nix
 export NIX_REMOTE=dummy://
 
 nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
+nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
 (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
 
@@ -14,7 +15,7 @@ fail=0
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --parse - < lang/$i.nix; then
+    if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
         echo "FAIL: $i shouldn't parse"
         fail=1
     fi
@@ -23,7 +24,7 @@ done
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
     i=$(basename $i .nix)
-    if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
+    if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
         echo "FAIL: $i should parse"
         fail=1
     fi
@@ -32,7 +33,7 @@ done
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --eval lang/$i.nix; then
+    if ! expect 1 nix-instantiate --eval lang/$i.nix; then
         echo "FAIL: $i shouldn't evaluate"
         fail=1
     fi
@@ -47,7 +48,7 @@ for i in lang/eval-okay-*.nix; do
         if test -e lang/$i.flags; then
             flags=$(cat lang/$i.flags)
         fi
-        if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
             echo "FAIL: $i should evaluate"
             fail=1
         elif ! diff lang/$i.out lang/$i.exp; then
@@ -57,7 +58,7 @@ for i in lang/eval-okay-*.nix; do
     fi
 
     if test -e lang/$i.exp.xml; then
-        if ! nix-instantiate --eval --xml --no-location --strict \
+        if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
                 lang/$i.nix > lang/$i.out.xml; then
             echo "FAIL: $i should evaluate"
             fail=1