Skip to content

Ngscheckmate fix

Ngscheckmate fix #5575

Workflow file for this run

name: test
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
on:
pull_request:
release:
types: [published]
merge_group:
types:
- checks_requested
branches:
- master
- dev
env:
NXF_ANSI_LOG: false
NFTEST_VER: "0.8.1"
# Cancel if a newer run is started
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
pytest-changes:
name: Check for changes (pytest)
runs-on: ubuntu-latest
outputs:
# Expose matched filters as job 'tags' output variable
tags: ${{ steps.filter.outputs.changes }}
steps:
- uses: actions/checkout@v4
- uses: frouioui/paths-filter@main
id: filter
with:
filters: "tests/config/pytesttags.yml"
token: ""
pytest:
name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
runs-on: ubuntu-latest
needs: pytest-changes
if: needs.pytest-changes.outputs.tags != '[]'
strategy:
fail-fast: false
matrix:
tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"]
profile: ["docker"]
# profile: ["docker", "singularity", "conda"]
TEST_DATA_BASE:
- "test-datasets/data"
NXF_VER:
- "23.04.0"
- "latest-everything"
env:
NXF_ANSI_LOG: false
TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
steps:
- name: Check out pipeline code
uses: actions/checkout@v4
- name: Hash Github Workspace
id: hash_workspace
run: |
echo "digest=$(echo sarek3_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.x"
cache: "pip"
cache-dependency-path: |
**/requirements.txt
- name: Install Python dependencies
run: pip install --upgrade -r tests/requirements.txt
- name: Install Nextflow ${{ matrix.NXF_VER }}
uses: nf-core/setup-nextflow@v1
with:
version: "${{ matrix.NXF_VER }}"
- name: Set up Singularity
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-singularity@v5
with:
singularity-version: 3.7.1
- name: Set up miniconda
if: matrix.profile == 'conda'
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}
- name: Cache test data
id: cache-testdata
uses: actions/cache@v3
with:
path: test-datasets/
key: ${{ steps.hash_workspace.outputs.digest }}
- name: Check out test data
if: steps.cache-testdata.outputs.cache-hit != 'true'
uses: actions/checkout@v4
with:
repository: nf-core/test-datasets
ref: sarek3
path: test-datasets/
- name: Replace remote paths in samplesheets
run: |
for f in tests/csv/3.0/*csv; do
sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/modules/=${{ github.workspace }}/test-datasets/=g" $f
echo "========== $f ============"
cat $f
echo "========================================"
done;
# Set up secrets
- name: Set up nextflow secrets
if: env.SENTIEON_LICENSE_BASE64 != null
run: |
nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
SENTIEON_AUTH_DATA=$(python bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
- name: Conda clean
if: matrix.profile == 'conda'
run: conda clean -a
- name: Run pytest-workflow
uses: Wandalen/wretry.action@v1
with:
command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
attempt_limit: 3
- name: Output log on failure
if: failure()
run: |
sudo apt install bat > /dev/null
batcat --decorations=always --color=always /home/runner/pytest_workflow_*/*/log.{out,err}
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v2
with:
name: logs-${{ matrix.profile }}
path: |
/home/runner/pytest_workflow_*/*/.nextflow.log
/home/runner/pytest_workflow_*/*/log.out
/home/runner/pytest_workflow_*/*/log.err
/home/runner/pytest_workflow_*/*/work
!/home/runner/pytest_workflow_*/*/work/conda
!/home/runner/pytest_workflow_*/*/work/singularity
nftest-changes:
name: Check for changes (nf-test)
runs-on: ubuntu-latest
outputs:
tags: ${{ steps.filter.outputs.changes }}
steps:
- uses: actions/checkout@v4
- name: Combine all tags.yml files
id: get_tags
run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
- name: debug
run: cat .github/tags.yml
- uses: frouioui/paths-filter@main
id: filter
with:
filters: ".github/tags.yml"
token: ""
nftest:
name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
runs-on: ubuntu-latest
needs: nftest-changes
if: needs.nftest-changes.outputs.tags != '[]'
strategy:
fail-fast: false
matrix:
tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"]
profile: ["docker"]
# profile: ["docker", "singularity", "conda"]
TEST_DATA_BASE:
- "test-datasets/data"
NXF_VER:
- "23.04.0"
- "latest-everything"
exclude:
- tags: "bwa/index"
- tags: "bwa/mem"
- tags: "cat/cat"
- tags: "cat/fastq"
- tags: "custom/dumpsoftwareversions"
- tags: "dragmap/align"
- tags: "fastp"
- tags: "fastqc"
- tags: "multiqc"
- tags: "samtools/stats"
- tags: "untar"
env:
NXF_ANSI_LOG: false
TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow
uses: nf-core/setup-nextflow@v1
- name: Cache nf-test installation
id: cache-software
uses: actions/cache@v3
with:
path: |
/usr/local/bin/nf-test
/home/runner/.nf-test/nf-test.jar
key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
- name: Install nf-test
if: steps.cache-software.outputs.cache-hit != 'true'
run: |
wget -qO- https://code.askimed.com/install/nf-test | bash
sudo mv nf-test /usr/local/bin/
- name: Setup apptainer
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-apptainer@main
- name: Set up Singularity
if: matrix.profile == 'singularity'
run: |
mkdir -p $NXF_SINGULARITY_CACHEDIR
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
- name: Set up miniconda
uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}
- name: Conda setup
run: |
conda clean -a
conda install -n base conda-libmamba-solver
conda config --set solver libmamba
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
echo $(realpath python) >> $GITHUB_PATH
# Set up secrets
- name: Set up nextflow secrets
if: env.SENTIEON_LICENSE_BASE64 != null
run: |
nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
# Test the module
- name: Run nf-test
run: |
nf-test test \
--profile=${{ matrix.profile }} \
--tag ${{ matrix.tags }} \
--tap=test.tap \
--verbose
confirm-pass:
runs-on: ubuntu-latest
needs:
- pytest
- nftest
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1
- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"