Add exit code tests

.github/workflows/test_exit-status.yml

@@ -0,0 +1,54 @@
+name: Test Exit Status of Commands
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'eask'
+      - '**.yml'
+      - lisp/**
+      - cmds/**
+      - src/**
+      - test/**
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+    - '**.md'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        emacs-version:
+          - 29.4
+
+    steps:
+    - uses: jcs090218/setup-emacs@master
+      with:
+        version: ${{ matrix.emacs-version }}
+
+    - uses: actions/checkout@v4
+
+    - name: Prepare Eask (Unix)
+      if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
+      run: |
+        chmod -R 777 ./
+        .github/scripts/setup-eask
+
+    - name: Prepare Eask (Windows)
+      if: matrix.os == 'windows-latest'
+      run: .github/scripts/setup-eask.ps1
+
+    - name: Testing...
+      run: |
+        make test-exit-status

Makefile

@@ -34,7 +34,7 @@ command-local:
 
 command-analyze:
 	./test/commands/analyze/dsl/run.sh
-	./test/commands/analyze/metadata/run.sh
+	./test/commands/analyze/metadata/run.sh;
 
 command-docker:
 	./test/commands/docker/run.sh
@@ -68,3 +68,24 @@ test-buttercup:
 
 test-ecukes:
 	./test/commands/test/ecukes/run.sh
+
+test-exit-status:
+	./test/exit-status/analyze/run.sh
+	./test/exit-status/clean/run.sh
+	./test/exit-status/compile/run.sh
+	./test/exit-status/create/run.sh
+	./test/exit-status/eval/run.sh
+	./test/exit-status/fixtures/run.sh
+	./test/exit-status/format/run.sh
+	./test/exit-status/generate/run.sh
+	./test/exit-status/install/run.sh
+	./test/exit-status/install-deps/run.sh
+	./test/exit-status/link/run.sh
+	./test/exit-status/lint/run.sh
+	./test/exit-status/load/run.sh
+	./test/exit-status/package/run.sh
+	./test/exit-status/refresh/run.sh
+	./test/exit-status/reinstall/run.sh
+	./test/exit-status/run/run.sh
+	./test/exit-status/test/run.sh
+	./test/exit-status/uninstall/run.sh

lisp/help/test/ert

@@ -1,4 +1,4 @@
 
-💡 You need to specify the file(s) you want the ert to run:
+💡 You need to specify the file(s) you want ert to run:
 
     $ eask test ert FILE-1 FILE-2

test/exit-status/analyze/Eask-error

@@ -0,0 +1,2 @@
+(package "" "" "")
+(scrog "unrecognized")

test/exit-status/analyze/Eask-normal

@@ -0,0 +1,2 @@
+(package "check-dsl" "0.0.1" "Test for DSL")
+(keywords "dsl")

test/exit-status/analyze/Eask-warn

@@ -0,0 +1,5 @@
+(package "check-dsl" "0.0.1" "Test for DSL")
+
+(keywords "dsl")
+
+(package-file "none.el")

test/exit-status/analyze/run.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2022-2024 the Eask authors.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+## Commentary:
+#
+# Tests "analyze" command's option handling and error behaviour.
+# See the tests in the test/checker directory too.
+
+set -e
+
+source ./test/fixtures/home/scripts/testing.sh
+
+cd $(dirname "$0")
+
+# Eask-normal - no errors or warnings
+# Eask-warn   - only warnings
+# Eask-error  - errors and  warnings
+
+should_run eask analyze Eask-normal
+
+should_run eask analyze Eask-warn
+
+should_error eask analyze Eask-error
+
+should_error eask analyze Eask-warn --strict
+
+# sanity check: flag should not change behavior in this case
+should_error eask analyze Eask-error --allow-error
+
+# Should report that Eask-normal was tested
+OUTPUT=$(should_error eask analyze --allow-error Eask-normal Eask-error)
+echo "$OUTPUT"
+should_match "(Checked 2 files)" "$OUTPUT"
+
+# Can also use a more concise form, but it doesn't print the output
+# should_match "(Checked 2 files)" "$(should_error eask analyze --allow-error Eask-normal Eask-error)"

test/exit-status/clean/Eask

@@ -0,0 +1 @@
+(package "check-clean" "0.0.1" "mock package")

test/exit-status/clean/mock.el

@@ -0,0 +1,5 @@
+;;;###autoload
+(defun my-ignore-fn ()
+  "Mock for testing"
+  (interactive)
+  (message "foo"))

test/exit-status/clean/run.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2022-2024 the Eask authors.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+## Commentary:
+#
+# Tests "clean" command's option handling and error behaviour.
+
+set -e
+
+source ./test/fixtures/home/scripts/testing.sh
+cd $(dirname "$0")
+
+echo "Preparing test"
+# clean elc should work
+eask compile mock.el
+if [ ! -f ./mock.elc ]; then
+    echo "failed to create .elc files"
+    exit 1
+fi
+
+echo "Test clean command"
+
+# partial input
+should_error eask clean
+
+should_run eask clean elc
+
+# dist is write protected
+# FIXME either or both of these should error
+# should_error eask clean dist
+# should_error eask clean dist --strict
+
+# recompile to use for --allow-error
+eask compile mock.el
+
+# FIXME
+# expect that clean dist fails and clean elc succeeds
+# should_error eask clean all --allow-error
+
+# if [ -f ./mock.elc ]; then
+#     echo "expected .elc files to be removed by eask clean all"
+#     exit 1
+# fi

test/exit-status/compile/Eask

@@ -0,0 +1 @@
+(package "check" "0.0.1" "mock package")

test/exit-status/compile/fail.el

@@ -0,0 +1,3 @@
+;; -*- lexical-binding: t -*-
+;; this must fail to compile
+(fail "" (

test/exit-status/compile/mock.el

@@ -0,0 +1,5 @@
+;;;###autoload
+(defun my-ignore-fn ()
+  "Mock for testing"
+  (interactive)
+  (message "foo"))

test/exit-status/compile/run.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2022-2024 the Eask authors.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+## Commentary:
+#
+# Tests "compile" command's option handling and error behaviour.
+
+set -e
+
+source ./test/fixtures/home/scripts/testing.sh
+cd $(dirname "$0")
+
+# FIXME this should match other commands which exit with status 1
+# partial input errors when there are no files
+# cd empty
+# should_error eask compile
+# cd ../
+
+# compiles with a warning
+should_run eask compile mock.el
+
+should_error eask compile --strict mock.el
+
+# --allow-error
+# rest state
+eask clean elc -v 0
+should_error eask compile fail.el
+should_error eask compile --allow-error fail.el mock.el
+
+# ensure mock.el is still compiled
+if [ ! -f mock.elc ]; then
+    echo "fail: expected to see mock.elc"
+    exit 1
+fi

test/exit-status/create/run.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2022-2024 the Eask authors.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+## Commentary:
+#
+# Tests "complete" command's error behaviour.
+
+set -e
+
+source ./test/fixtures/home/scripts/testing.sh
+cd $(dirname "$0")
+
+# partial input errors
+should_error eask create
+should_error eask create elpa
+should_error eask create package
+
+# FIXME
+# existing file errors
+# should_error eask create elpa foo
+# should_error eask create package foo

test/exit-status/eval/Eask

@@ -0,0 +1 @@
+(package "check" "0.0.1" "mock package")

test/exit-status/eval/run.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2022-2024 the Eask authors.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+## Commentary:
+#
+# Tests "eval" command's option handling and error behaviour.
+
+set -e
+
+source ./test/fixtures/home/scripts/testing.sh
+cd $(dirname "$0")
+
+# partial input
+should_error eask eval
+
+should_run eask eval '(message "hello")'
+
+should_run eask eval '(warn "hello")'
+
+# FIXME
+# should_error eask eval '(warn "hello")' --strict
+
+should_error eask eval '(error "hello")'
+should_error eask eval "(signal 'error \"hello\")"
+# --allow-error doesn't really change anything here
+should_error eask eval '(error "hello")' --allow-error

test/exit-status/fixtures/valid/Eask

@@ -0,0 +1,6 @@
+(package "valid-package" "0.0.1" "mock package that installs cleanly")
+
+(source 'gnu)
+(source 'melpa)
+
+(depends-on "emacs" "28.3")

test/exit-status/fixtures/valid/foo.el

@@ -0,0 +1,6 @@
+(defun foo-nothing (args)
+  "docstring"
+  (interactive "P")
+  )
+
+(provide 'foo)

test/exit-status/fixtures/valid/valid-package-pkg.el

@@ -0,0 +1,2 @@
+(define-package "valid-package" "0.0.1"
+  "mock package that installs cleanly" '((emacs "28.3")))

test/exit-status/format/Eask

@@ -0,0 +1 @@
+(package "check" "0.0.1" "mock package")

test/exit-status/format/fail.el

@@ -0,0 +1,3 @@
+;; -*- lexical-binding: t -*-
+;; this must fail to compile
+(fail "" (

test/exit-status/format/mock.el

@@ -0,0 +1,5 @@
+;;;###autoload
+(defun my-ignore-fn ()
+  "Mock for testing"
+  (interactive)
+  (message "foo"))