Skip to content

Commit

Permalink
Merge pull request #736 from maxulysse/dev_complete_CI
Browse files Browse the repository at this point in the history 
simplify default tests + add conda/singularity
  • Loading branch information
@maxulysse
maxulysse authored Sep 8, 2022
2 parents f7c4caf + 349e1d6 commit f944d43
Show file tree
Hide file tree
Showing 8 changed files with 191 additions and 151 deletions.
50 changes: 43 additions & 7 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,28 +26,63 @@ jobs:
- "latest-everything"
test:
- "default"
profile: ["docker"]
profile: ["docker", "singularity", "conda"]
exclude:
- NXF_VER: "latest-everything"
profile: "conda"
steps:
- name: Check out pipeline code
uses: actions/checkout@v2

- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: "3.x"

- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install Python dependencies
run: python -m pip install --upgrade pip pytest-workflow

- uses: actions/cache@v2
with:
path: /usr/local/bin/nextflow
key: ${{ runner.os }}
restore-keys: |
${{ runner.os }}-nextflow-
- name: Install Nextflow
uses: nf-core/setup-nextflow@v1
with:
version: "${{ matrix.NXF_VER }}"

- name: Set up Python
uses: actions/setup-python@v2
- name: Set up Singularity
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-singularity@v5
with:
python-version: "3.x"
singularity-version: 3.7.1

- name: Install dependencies
run: python -m pip install --upgrade pip pytest-workflow
- name: Set up miniconda
if: matrix.profile == 'conda'
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}

- name: Conda clean
if: matrix.profile == 'conda'
run: conda clean -a

- name: Run pipeline with tests settings
uses: Wandalen/wretry.action@v1.0.11
with:
command: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes
command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes
attempt_limit: 3

- name: Output log on failure
Expand Down Expand Up @@ -91,6 +126,7 @@ jobs:
- "freebayes"
- "gatk4_spark"
- "haplotypecaller"
- "intervals"
- "manta"
- "markduplicates"
- "mpileup"
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- [#731](https://github.com/nf-core/sarek/pull/731) - Run the somatic test as default on `-profile test_full`, the germline can be tested with `-profile test_full_somatic`
- [#733](https://github.com/nf-core/sarek/pull/733) - Add description for params.cf_chrom_len
- [#734](https://github.com/nf-core/sarek/pull/734) - nf-core modules update -a
- [#736](https://github.com/nf-core/sarek/pull/736) - More extensive CI for default test

### Fixed

Expand Down
43 changes: 23 additions & 20 deletions conf/test.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,29 +253,32 @@ process {
}
}

// ENABLE CI containers for testing
// Enable container engines/virtualisation envs for CI testing
// only works when specified with the profile ENV
// otherwise tests can be done with the regular provided profiles
if (System.getenv('PROFILE')) {
if ("$PROFILE" == "conda") {
params.enable_conda = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
conda.createTimeout = "120 min"
params.enable_conda = true
charliecloud.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
singularity.enabled = false
} else if ("$PROFILE" == "docker") {
docker.enabled = true
docker.userEmulation = { params.use_gatk_spark ? false : true }.call()
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
fixOwnership = true
docker.enabled = true
docker.userEmulation = { params.use_gatk_spark ? false : true }.call()
charliecloud.enabled = false
podman.enabled = false
shifter.enabled = false
singularity.enabled = false
} else if ("$PROFILE" == "singularity") {
singularity.enabled = true
singularity.autoMounts = true
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
params.singularity_pull_docker_container = false
singularity.autoMounts = true
singularity.enabled = true
charliecloud.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
}
}
31 changes: 15 additions & 16 deletions nextflow.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,63 +143,62 @@ profiles {
debug { process.beforeScript = 'echo $HOSTNAME' }
conda {
params.enable_conda = true
charliecloud.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
singularity.enabled = false
}
mamba {
params.enable_conda = true
conda.useMamba = true
params.enable_conda = true
charliecloud.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
singularity.enabled = false
}
docker {
docker.enabled = true
docker.userEmulation = { params.use_gatk_spark ? false : true }.call()
singularity.enabled = false
charliecloud.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
fixOwnership = true
singularity.enabled = false
}
singularity {
singularity.enabled = true
singularity.autoMounts = true
singularity.enabled = true
charliecloud.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
podman {
podman.enabled = true
charliecloud.enabled = false
docker.enabled = false
singularity.enabled = false
shifter.enabled = false
charliecloud.enabled = false
singularity.enabled = false
}
shifter {
shifter.enabled = true
charliecloud.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
charliecloud.enabled = false
singularity.enabled = false
}
charliecloud {
charliecloud.enabled = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
singularity.enabled = false
}
gitpod {
executor.name = 'local'
executor.cpus = 16
executor.memory = 60.GB
executor.name = 'local'
}
test { includeConfig 'conf/test.config' }
test_full { includeConfig 'conf/test_full.config' }
Expand Down
63 changes: 3 additions & 60 deletions tests/test_default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
- path: results/preprocessing/recalibrated/test/test.recal.cram.crai
# binary changes md5sums on reruns.
- path: results/reports/bcftools/strelka/test/test.strelka.variants.bcftools_stats.txt
md5sum: 28d99fd4fa63e24e6805f1d12c3b71d3
# conda changes md5sums for test.
- path: results/reports/fastqc/test-test_L1
- path: results/reports/markduplicates/test/test.md.metrics
contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"]
Expand All @@ -49,9 +49,9 @@
- path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
md5sum: b1c2a861f64e20a94108a6de3b76c582
- path: results/reports/samtools/test/test.md.cram.stats
md5sum: 52411f93d9b5382342b40a351c2e313c
# conda changes md5sums for test.
- path: results/reports/samtools/test/test.recal.cram.stats
md5sum: 811d834ff8849e69e7c5925dc2a34257
# conda changes md5sums for test.
- path: results/reports/vcftools/strelka/test/test.strelka.variants.FILTER.summary
md5sum: dd87f507da7de20d5318841af312493b
- path: results/reports/vcftools/strelka/test/test.strelka.variants.TsTv.count
Expand All @@ -67,60 +67,3 @@
- path: results/variant_calling/strelka/test/test.strelka.variants.vcf.gz.tbi
- path: results/strelka
should_exist: false
- name: Run default pipeline without intervals
command: nextflow run main.nf -profile test,no_intervals
tags:
- default
- no_intervals
- preprocessing
files:
- path: results/csv/markduplicates.csv
md5sum: 0d6120bb99e92f6810343270711ca53e
- path: results/csv/markduplicates_no_table.csv
md5sum: 2a2d3d4842befd4def39156463859ee3
- path: results/csv/recalibrated.csv
md5sum: 42628ec994c16f565e5407b40a9c1ac3
- path: results/multiqc
- path: results/no_intervals.bed
md5sum: f3dac01ea66b95fe477446fde2d31489
- path: results/no_intervals.bed.gz
md5sum: f3dac01ea66b95fe477446fde2d31489
- path: results/no_intervals.bed.gz.tbi
md5sum: f3dac01ea66b95fe477446fde2d31489
- path: results/preprocessing/markduplicates/test/test.md.cram
# binary changes md5sums on reruns.
- path: results/preprocessing/markduplicates/test/test.md.cram.crai
# binary changes md5sums on reruns.
- path: results/preprocessing/recal_table/test/test.recal.table
md5sum: 4ac774bf5f1157e77426fd82f5ac0fbe
- path: results/preprocessing/recalibrated/test/test.recal.cram
# binary changes md5sums on reruns.
- path: results/preprocessing/recalibrated/test/test.recal.cram.crai
# binary changes md5sums on reruns.
- path: results/reports/fastqc/test-test_L1
- path: results/reports/markduplicates/test/test.md.metrics
contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"]
- path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt
md5sum: 76fa71922a3f748e507c2364c531dfcb
- path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt
md5sum: abc5df85e302b79985627888870882da
- path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt
md5sum: d536456436eb275159b8c6af83213d80
- path: results/reports/mosdepth/test/test.md.regions.bed.gz
md5sum: 38fe39894abe62e38f8ac214cba64f2b
- path: results/reports/mosdepth/test/test.md.regions.bed.gz.csi
md5sum: b1c2a861f64e20a94108a6de3b76c582
- path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt
md5sum: 76fa71922a3f748e507c2364c531dfcb
- path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt
md5sum: abc5df85e302b79985627888870882da
- path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt
md5sum: d536456436eb275159b8c6af83213d80
- path: results/reports/mosdepth/test/test.recal.regions.bed.gz
md5sum: 38fe39894abe62e38f8ac214cba64f2b
- path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi
md5sum: b1c2a861f64e20a94108a6de3b76c582
- path: results/reports/samtools/test/test.md.cram.stats
md5sum: 52411f93d9b5382342b40a351c2e313c
- path: results/reports/samtools/test/test.recal.cram.stats
md5sum: d80ad11b8e308a3d5b1fb6f8e13cef38
Loading

0 comments on commit f944d43

Please sign in to comment.