Overhaul GHA

.github/workflows/awstest.yml

@@ -4,10 +4,24 @@ name: nf-core AWS test
 
 on:
   workflow_dispatch:
+    inputs:
+      profiletest:
+        description: "Trigger profile tests (smaller) on AWS"
+        type: boolean
+        default: true
+      somatic:
+        description: "Trigger somatic full test on AWS"
+        type: boolean
+        default: false
+      germline:
+        description: "Trigger germline full test on AWS"
+        type: boolean
+        default: false
+
 jobs:
-  run-tower:
+  trigger-profile-test:
     name: Run AWS tests
-    if: github.repository == 'nf-core/sarek'
+    if: github.repository == 'nf-core/sarek' || inputs.profiletest == 'true'
     runs-on: ubuntu-latest
     steps:
       # Launch workflow using Tower CLI tool action
@@ -26,7 +40,60 @@ jobs:
           profiles: test
       - uses: actions/upload-artifact@v3
         with:
-          name: Tower debug log file
+          name: tower-profiletest-log
           path: |
             tower_action_*.log
             tower_action_*.json
+
+  trigger-full-test-somatic:
+    name: Run AWS full tests
+    if: ( github.repository == 'nf-core/sarek' && github.event_name == 'release' ) || inputs.somatic == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      # Launch workflow using Tower CLI tool action
+      - name: Launch workflow via tower
+        uses: seqeralabs/action-tower-launch@v2
+        with:
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/somatic_test
+          parameters: |
+            {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/somatic_test"
+            }
+          profiles: test_full
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: tower-full-somatic-log
+          path: |
+            tower_action_*.log
+            tower_action_*.json
+
+  trigger-full-test-germline:
+    name: Run AWS full tests
+    if: ( github.repository == 'nf-core/sarek' && github.event_name == 'release' ) || inputs.germline == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      # Launch workflow using Tower CLI tool action
+      - name: Launch workflow via tower
+        uses: seqeralabs/action-tower-launch@v2
+        with:
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/germline_test
+          parameters: |
+            {
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/germline_test"
+            }
+          profiles: test_full_germline
+      - uses: actions/upload-artifact@v3
+        with:
+          name: tower-full-germline-log
+          path: tower_action_*.log

.github/workflows/pytest-workflow.yml

@@ -1,8 +1,18 @@
-name: pytest-workflow
+name: test
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
   pull_request:
-    branches: [dev]
+    branches:
+      - master
+      - dev
+  release:
+    types:
+      - published
+  merge_group:
+    types:
+      - checks_requested
+    branches:
+      - master
 
 # Cancel if a newer run is started
 concurrency:
@@ -32,32 +42,13 @@ jobs:
       fail-fast: false
       matrix:
         tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
-        profile: ["docker"]
-        # profile: ["docker", "singularity", "conda"]
+        profile: ["docker", "singularity"]
         TEST_DATA_BASE:
           - "test-datasets/data"
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
         exclude:
-          # - profile: "conda"
-          #   tags: concatenate_vcfs
-          # - profile: "conda"
-          #   tags: deepvariant
-          # - profile: "conda"
-          #   tags: haplotypecaller
-          # - profile: "conda"
-          #   tags: merge
-          # - profile: "conda"
-          #   tags: snpeff
-          # - profile: "conda"
-          #   tags: umi
-          # - profile: "conda"
-          #   tags: validation_checks
-          # - profile: "conda"
-          #   tags: vep
-          # - profile: "conda"
-          #  tags: sentieon/bwamem
           - profile: "singularity"
             tags: concatenate_vcfs
           - profile: "singularity"
@@ -107,14 +98,14 @@ jobs:
           python-version: "3.x"
 
       - uses: actions/cache@v3
+        id: python-cache-setup
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
 
       - name: Install Python dependencies
-        run: python -m pip install --upgrade pip pytest-workflow cryptography
+        if: steps.python-cache-setup.outputs.cache-hit != 'true'
+        run: python -m pip install --upgrade -r tests/requirements.txt
 
       - name: Install Nextflow ${{ matrix.NXF_VER }}
         uses: nf-core/setup-nextflow@v1
@@ -175,3 +166,22 @@ jobs:
             /home/runner/pytest_workflow_*/*/work
             !/home/runner/pytest_workflow_*/*/work/conda
             !/home/runner/pytest_workflow_*/*/work/singularity
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"