Merge branch 'dev' into maps

nf-core · Nov 7, 2023 · 87274c3 · 87274c3
2 parents bc28386 + dc9717a
commit 87274c3
Show file tree

Hide file tree

Showing 361 changed files with 4,279 additions and 979 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,3 @@
-*   @FriederikeHanssen
-*   @maxulysse
+* @FriederikeHanssen @maxulysse
+*.nf.test* @nf-core/nf-test
+.github/workflows/ @nf-core/a-team
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,34 +11,40 @@ on:
       - master
       - dev
 
+env:
+  NXF_ANSI_LOG: false
+  NFTEST_VER: "0.8.1"
+
 # Cancel if a newer run is started
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  changes:
-    name: Check for changes
+  pytest-changes:
+    name: Check for changes (pytest)
     runs-on: ubuntu-latest
     outputs:
       # Expose matched filters as job 'tags' output variable
       tags: ${{ steps.filter.outputs.changes }}
     steps:
       - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+
+      - uses: frouioui/paths-filter@main
         id: filter
         with:
-          filters: "tests/config/tags.yml"
+          filters: "tests/config/pytesttags.yml"
+          token: ""
 
-  test:
+  pytest:
     name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
     runs-on: ubuntu-latest
-    needs: changes
-    if: needs.changes.outputs.tags != '[]'
+    needs: pytest-changes
+    if: needs.pytest-changes.outputs.tags != '[]'
     strategy:
       fail-fast: false
       matrix:
-        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
+        tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"]
         profile: ["docker"]
         # profile: ["docker", "singularity", "conda"]
         TEST_DATA_BASE:
@@ -50,6 +56,7 @@ jobs:
       NXF_ANSI_LOG: false
       TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
       SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
@@ -154,10 +161,135 @@ jobs:
             !/home/runner/pytest_workflow_*/*/work/conda
             !/home/runner/pytest_workflow_*/*/work/singularity
 
+  nftest-changes:
+    name: Check for changes (nf-test)
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.filter.outputs.changes }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Combine all tags.yml files
+        id: get_tags
+        run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
+
+      - name: debug
+        run: cat .github/tags.yml
+
+      - uses: frouioui/paths-filter@main
+        id: filter
+        with:
+          filters: ".github/tags.yml"
+          token: ""
+
+  nftest:
+    name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
+    runs-on: ubuntu-latest
+    needs: nftest-changes
+    if: needs.nftest-changes.outputs.tags != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"]
+        profile: ["docker"]
+        # profile: ["docker", "singularity", "conda"]
+        TEST_DATA_BASE:
+          - "test-datasets/data"
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
+        exclude:
+          - tags: "bwa/index"
+          - tags: "bwa/mem"
+          - tags: "cat/cat"
+          - tags: "cat/fastq"
+          - tags: "custom/dumpsoftwareversions"
+          - tags: "fastp"
+          - tags: "fastqc"
+          - tags: "samtools/stats"
+    env:
+      NXF_ANSI_LOG: false
+      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
+      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "temurin"
+          java-version: "17"
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v1
+
+      - name: Cache nf-test installation
+        id: cache-software
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Setup apptainer
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-apptainer@main
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        run: |
+          mkdir -p $NXF_SINGULARITY_CACHEDIR
+          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+      - name: Set up miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: "latest"
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+          python-version: ${{ matrix.python-version }}
+
+      - name: Conda setup
+        run: |
+          conda clean -a
+          conda install -n base conda-libmamba-solver
+          conda config --set solver libmamba
+          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
+          echo $(realpath python) >> $GITHUB_PATH
+
+      # Set up secrets
+      - name: Set up nextflow secrets
+        if: env.SENTIEON_LICENSE_BASE64 != null
+        run: |
+          nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+          nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
+          SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
+          SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
+          SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
+          SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
+          nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
+
+      # Test the module
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --profile=${{ matrix.profile }} \
+            --tag ${{ matrix.tags }} \
+            --tap=test.tap \
+            --verbose
+
   confirm-pass:
     runs-on: ubuntu-latest
     needs:
-      - test
+      - pytest
+      - nftest
     if: always()
     steps:
       - name: All tests ok

diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,12 @@
 *.code-workspace
+*.pyc
 .nextflow*
-work/
-data/
-results/
 .DS_Store
-testing/
+.nf-test*
+.nf-test/
+data/
 testing*
-*.pyc
+testing/
 test-datasets/
+results/
+work/
diff --git a/.nf-core.yml b/.nf-core.yml
@@ -5,6 +5,7 @@ lint:
     - .github/workflows/awsfulltest.yml
     - conf/modules.config
   files_unchanged:
+    - .gitignore
     - assets/nf-core-sarek_logo_light.png
     - docs/images/nf-core-sarek_logo_dark.png
     - docs/images/nf-core-sarek_logo_light.png

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,22 +9,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- [#1113](https://github.com/nf-core/sarek/pull/1113) - Adding CNVkit genemetrics module
 - [#1193](https://github.com/nf-core/sarek/pull/1193) - Adding support for Sentieon's DnaScope for germline variant-calling including joint-germline
 - [#1271](https://github.com/nf-core/sarek/pull/1271) - Back to dev
+- [#1290](https://github.com/nf-core/sarek/pull/1290) - Add nf-test for whole pipeline.
 
 ### Changed
 
+- [#1278](https://github.com/nf-core/sarek/pull/1278) - Hide sentieon parameters similar to other variant callers
 - [#1280](https://github.com/nf-core/sarek/pull/1280) - Replacing link to `SentieonDNAscopeModel1.1.model` in Sentieon's S3 with link to same file in igenomes' S3
+- [#1303](https://github.com/nf-core/sarek/pull/1303) - Ressurect vep_version params and changed its scope to pipeline to enable usage for vep loftee plugin
+- [#1304](https://github.com/nf-core/sarek/pull/1304) - Update modules
+- [#1311](https://github.com/nf-core/sarek/pull/1311) - Update local modules with an `environment.yml` file
 - [#1317](https://github.com/nf-core/sarek/pull/1317) - Add new tools to subway map
 
 ### Fixed
 
 - [#1277](https://github.com/nf-core/sarek/pull/1277) - Fix null value issue for Mutect2 joint calling
+- [#1287](https://github.com/nf-core/sarek/pull/1287) - Adding label `process_single` to local modules
+- [#1298](https://github.com/nf-core/sarek/pull/1298) - Fix annotation cache usage
+- [#1301](https://github.com/nf-core/sarek/pull/1301) - Fix nf-prov usage
+- [#1315](https://github.com/nf-core/sarek/pull/1315) - Avoid clash of configs of `FILTERVARIANTTRANCHES` in the Sentieon-Haplotyper and GATK-Haplotypecaller subworkflows
+
+### Removed
+
+- [#1298](https://github.com/nf-core/sarek/pull/1298) - Remove `--use_annotation_cache_keys` params
 
 ### Dependencies
 
 | Dependency | Old version | New version |
 | ---------- | ----------- | ----------- |
+| fastqc     | 0.11.9      | 0.12.1      |
+| multiqc    | 1.15        | 1.17        |
+
+### Modules / Subworkflows
+
+| script                        | Old name                      | New name                      |
+| ----------------------------- | ----------------------------- | ----------------------------- |
+| `gatk4spark/applybqsr`        | `GATK4_APPLYBQSRSPARK`        | `GATK4SPARK_APPLYBQSR`        |
+| `gatk4spark/baserecalibrator` | `GATK4_BASERECALIBRATORSPARK` | `GATK4SPARK_BASERECALIBRATOR` |
+| `gatk4spark/markduplicates`   | `GATK4_MARKDUPLICATESSPARK`   | `GATK4SPARK_MARKDUPLICATES`   |
 
 ## [3.3.2](https://github.com/nf-core/sarek/releases/tag/3.3.2) - Ráhpajávvre
 
@@ -59,6 +83,7 @@ A lake near the Rapaselet delta.
 ### Added
 
 - [#1231](https://github.com/nf-core/sarek/pull/1231) - Back to dev
+- [#1244](https://github.com/nf-core/sarek/pull/1244) - Add bcf annotate module
 
 ### Changed
 
@@ -93,6 +118,7 @@ Rapaselet is a delta formed by the Rapaätno river between the Bielloriehppe mas
 - [#1173](https://github.com/nf-core/sarek/pull/1173) - CI tests for VQSR track with stub runs
 - [#1122](https://github.com/nf-core/sarek/pull/1122), [#1196](https://github.com/nf-core/sarek/pull/1196) - Add `annotation cache` functionality
 - [#1184](https://github.com/nf-core/sarek/pull/1184) - Stub-based CI-test of Sentieon joint-germline variant-calling with VQSR
+- [#1288](https://github.com/nf-core/sarek/pull/1288) - Add nf-test continuous integration.
 
 ### Changed
 

diff --git a/README.md b/README.md
@@ -50,7 +50,7 @@ Depending on the options and samples provided, the pipeline can currently perfor
   - `Sentieon Haplotyper`
   - `Strelka2`
   - `TIDDIT`
-- Variant filtering and annotation (`SnpEff`, `Ensembl VEP`)
+- Variant filtering and annotation (`SnpEff`, `Ensembl VEP`, `BCFtools annotate`)
 - Summarise and represent QC (`MultiQC`)
 
 <p align="center">
@@ -131,6 +131,7 @@ We thank the following people for their extensive assistance in the development
 - [Francesco Lescai](https://github.com/lescai)
 - [Gavin Mackenzie](https://github.com/GCJMackenzie)
 - [Gisela Gabernet](https://github.com/ggabernet)
+- [Grant Neilson](https://github.com/grantn5)
 - [gulfshores](https://github.com/gulfshores)
 - [Harshil Patel](https://github.com/drpatelh)
 - [James A. Fellows Yates](https://github.com/jfy133)

diff --git a/conf/base.config b/conf/base.config
@@ -70,11 +70,15 @@ process {
         cpus            = { check_max( 24 * task.attempt, 'cpus' ) }
         memory          = { check_max( 30.GB * task.attempt, 'memory' ) }
     }
-    withName: 'GATK4_MARKDUPLICATES|GATK4_MARKDUPLICATESSPARK' {
+    withName:'CNVKIT_BATCH' {
+        label           = "process_high"
+        memory          = { check_max( 36.GB * task.attempt, 'memory' ) }
+    }
+    withName: 'GATK4_MARKDUPLICATES|GATK4SPARK_MARKDUPLICATES' {
         cpus           = { check_max( 6 * task.attempt, 'cpus' ) }
         memory         = { check_max( 30.GB * task.attempt, 'memory' ) }
     }
-    withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_BASERECALIBRATOR_SPARK|GATK4_GATHERBQSRREPORTS'{
+    withName:'GATK4_APPLYBQSR|GATK4SPARK_APPLYBQSR|GATK4_BASERECALIBRATOR|GATK4SPARK_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS'{
         cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
         memory          = { check_max( 4.GB * task.attempt, 'memory' ) }
     }

diff --git a/conf/modules/annotate.config b/conf/modules/annotate.config
@@ -37,7 +37,7 @@ process {
             ext.args         = { [
                 (params.vep_dbnsfp && params.dbnsfp && !params.dbnsfp_consequence)    ? "--plugin dbNSFP,${params.dbnsfp.split("/")[-1]},${params.dbnsfp_fields}"                                              : '',
                 (params.vep_dbnsfp && params.dbnsfp && params.dbnsfp_consequence)     ? "--plugin dbNSFP,'consequence=${params.dbnsfp_consequence}',${params.dbnsfp.split("/")[-1]},${params.dbnsfp_fields}"   : '',
-                (params.vep_loftee)                                                   ? "--plugin LoF,loftee_path:/opt/conda/envs/nf-core-vep-${params.vep_version}/share/ensembl-vep-${params.vep_version}-0" : '',
+                (params.vep_loftee)                                                   ? "--plugin LoF,loftee_path://usr/local/share/ensembl-vep-${params.vep_version}" : '',
                 (params.vep_spliceai && params.spliceai_snv && params.spliceai_indel) ? "--plugin SpliceAI,snv=${params.spliceai_snv.split("/")[-1]},indel=${params.spliceai_indel.split("/")[-1]}"            : '',
                 (params.vep_spliceregion)                                             ? '--plugin SpliceRegion'                                                                                                : '',
                 (params.vep_out_format)                                               ? "--${params.vep_out_format}"                                                                                           : '--vcf',
@@ -60,6 +60,19 @@ process {
         }
     }
 
+    // BCFTOOLS ANNOTATE
+    if (params.tools && params.tools.split(',').contains('bcfann')) {
+        withName: 'NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:VCF_ANNOTATE_BCFTOOLS:BCFTOOLS_ANNOTATE' {
+            ext.args         = '--output-type z'
+            ext.prefix       = { input.baseName - ".vcf" + "_BCF.ann" }
+            publishDir       = [
+                    mode: params.publish_dir_mode,
+                    path: { "${params.outdir}/annotation/${meta.variantcaller}/${meta.id}/" },
+                    pattern: "*{gz}"
+                ]
+        }
+    }
+
     // SNPEFF THEN VEP
     if (params.tools && params.tools.split(',').contains('merge')) {
         withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:VCF_ANNOTATE_MERGE:ENSEMBLVEP_VEP" {
@@ -69,7 +82,7 @@ process {
     }
 
     // ALL ANNOTATION TOOLS
-    if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('vep') || params.tools.split(',').contains('merge'))) {
+    if (params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('vep') || params.tools.split(',').contains('merge') || params.tools.split(',').contains('bcfann'))) {
         withName: "NFCORE_SAREK:SAREK:VCF_ANNOTATE_ALL:.*:(TABIX_BGZIPTABIX|TABIX_TABIX)" {
             ext.prefix       = { input.name - ".vcf" }
             publishDir       = [

diff --git a/conf/modules/cnvkit.config b/conf/modules/cnvkit.config
@@ -47,4 +47,14 @@ process {
             pattern: "*{bed,cnn,cnr,cns,pdf,png}"
         ]
     }
+    // CNVKIT
+    withName: 'CNVKIT_GENEMETRICS' {
+        ext.prefix       = { "${cnr.baseName}.genemetrics" }
+        ext.when         = { params.tools && params.tools.split(',').contains('cnvkit') }
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/variant_calling/cnvkit/${meta.id}/" },
+            pattern: "*{tsv}"
+        ]
+    }
 }
diff --git a/conf/modules/haplotypecaller.config b/conf/modules/haplotypecaller.config
@@ -43,7 +43,7 @@ process {
         ]
     }
 
-    withName: 'FILTERVARIANTTRANCHES' {
+    withName: '.*:VCF_VARIANT_FILTERING_GATK:FILTERVARIANTTRANCHES' {
         ext.prefix       = {"${meta.id}.haplotypecaller"}
         ext.args         = { "--info-key CNN_1D" }
         publishDir       = [

diff --git a/conf/modules/markduplicates.config b/conf/modules/markduplicates.config
@@ -90,7 +90,7 @@ process {
         ]
     }
 
-    withName: 'GATK4_MARKDUPLICATES_SPARK' {
+    withName: 'GATK4SPARK_MARKDUPLICATES' {
         ext.args         = '--remove-sequencing-duplicates false -VS LENIENT'
         ext.prefix       = { "${meta.id}.md.cram" }
         publishDir       = [