add test for failing tool

.github/workflows/tools.yaml

@@ -145,8 +145,8 @@ jobs:
         if ! grep -q "tool1.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool1.xml must be in the repo list"; exit 1; fi
         if ! grep -q "tool2.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool2.xml must be in the repo list"; exit 1; fi
     # chunk-count will only consider tools/workflows depending on the workflow input to the action
-    - name: Check that the number of chunks is three, i.e. the number of tools
-      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "3" ]; then exit 1; fi
+    - name: Check that the number of chunks is four, i.e. the number of tools
+      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "4" ]; then exit 1; fi
 
   lint:
     name: Test linting of tools
@@ -270,13 +270,30 @@ jobs:
         mode: combine
         html-report: true
     - name: Check statistics
-      run: if ! grep -q "3\s\+success" <<<$(echo ${{ steps.combine.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
+      run: |
+        if ! grep -q "3\s\+success" <<<$(echo "${{ steps.combine.outputs.statistics }}"); then
+          echo "wrong statistics: expected 3 success";
+          echo "${{ steps.combine.outputs.statistics }}";
+          exit 1;
+        fi
+        if ! grep -q "1\s\+failure" <<<$(echo "${{ steps.combine.outputs.statistics }}"); then
+          echo "wrong statistics: expected 1 failure";
+          echo "${{ steps.combine.outputs.statistics }}";
+          exit 1;
+        fi
     - uses: actions/upload-artifact@v3
       with:
         name: 'All tool test results'
         path: upload
     - name: Check outputs
       uses: ./
+      continue-on-error: true
       id: check
       with:
         mode: check
+    - name: Check on failures
+      if: steps.check.outcome != 'failure'
+      run: |
+        echo "outcome  ${{ steps.check.outcome }}" 
+        exit 1
+

test/tools/fail/.shed.yml

@@ -0,0 +1,4 @@
+categories: [Sequence Analysis]
+description: just fail
+name: fail
+owner: adent

test/tools/fail/galaxy_slots.xml

@@ -0,0 +1,35 @@
+<tool id="fail" name="fail" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="20.01" license="MIT">
+    <macros>
+        <token name="@TOOL_VERSION@">0.1.0</token>
+        <token name="@VERSION_SUFFIX@">0</token>
+    </macros>
+    <command detect_errors="exit_code"><![CDATA[
+        touch '$out' &&
+        exit 1
+    ]]></command>
+    <inputs/>
+    <outputs>
+        <data name="out" format="txt"/>
+    </outputs>
+    <tests>
+        <test>
+            <output name="out">
+                <assert_contents>
+                    <has_n_lines n="0"/>
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+    <help><![CDATA[
+
+**Input**
+
+
+**Output**
+
+
+    ]]></help>
+    <citations>
+        <citation type="doi">blah</citation>
+    </citations>
+</tool>