Refactor

CHANGELOG.md

@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- [#1151](https://github.com/nf-core/sarek/pull/1151) - Refactor codebase
+
 ### Fixed
 
 - [#1143](https://github.com/nf-core/sarek/pull/1143) - `snpeff_db` is now a string

README.md

@@ -34,7 +34,7 @@ It's listed on [Elixir - Tools and Data Services Registry](https://bio.tools/nf-
 Depending on the options and samples provided, the pipeline can currently perform the following:
 
 - Form consensus reads from UMI sequences (`fgbio`)
-- Sequencing quality control and trimming (`FastQC`, `fastp`)
+- Sequencing quality control and trimming (enabled by `--trim_fastq`) (`FastQC`, `fastp`)
 - Map Reads to Reference (`BWA-mem`, `BWA-mem2`, `dragmap` or `Sentieon BWA-mem`)
 - Process BAM file (`GATK MarkDuplicates`, `GATK BaseRecalibrator` and `GATK ApplyBQSR` or `Sentieon LocusCollector` and `Sentieon Dedup`)
 - Summarise alignment statistics (`samtools stats`, `mosdepth`)

conf/modules/aligner.config

@@ -33,13 +33,7 @@ process {
             ext.when         = { params.aligner == "sentieon-bwamem" }
         }
 
-
-
-        withName: "(BWAMEM.*_MEM|DRAGMAP_ALIGN)" {
-            // Markduplicates Spark NEEDS name-sorted reads or runtime goes through the roof
-            // However if it's skipped, reads need to be coordinate-sorted
-            // Only name sort if Spark for Markduplicates + duplicate marking is not skipped
-            ext.args2        = { params.use_gatk_spark && params.use_gatk_spark.contains('markduplicates') && (!params.skip_tools || (params.skip_tools && !params.skip_tools.split(',').contains('markduplicates'))) ? '-n' : '' }
+        withName: "(BWAMEM.*_MEM|DRAGMAP_ALIGN|SENTIEON_BWAMEM)" {
             ext.prefix       = { params.split_fastq > 1 ? "${meta.id}".concat('.').concat(reads.get(0).name.tokenize('.')[0]) : "${meta.id}.sorted" }
             publishDir       = [
                 mode: params.publish_dir_mode,
@@ -61,29 +55,12 @@ process {
             ]
         }
 
-
-        withName: "SENTIEON_BWAMEM" {
-            // Markduplicates Spark NEEDS name-sorted reads or runtime goes through the roof.
-            // However, currently SENTIEON_BWAMEM only supports coordinate sorting the reads.
-            ext.prefix       = { params.split_fastq > 1 ? "${meta.id}".concat('.').concat(reads.get(0).name.tokenize('.')[0]) : "${meta.id}.sorted" }
-            publishDir       = [
-                mode: params.publish_dir_mode,
-                path: { "${params.outdir}/preprocessing/" },
-                pattern: "*bam",
-                // Only save if save_output_as_bam AND
-                // (save_mapped OR no_markduplicates OR sentieon_dedup) AND
-                // only a single BAM file per sample
-                saveAs: {
-                    if (params.save_output_as_bam &&
-                        (
-                            params.save_mapped ||
-                            (params.skip_tools && params.skip_tools.split(',').contains('markduplicates')) &&
-                            !(params.tools && params.tools.split(',').contains('sentieon_dedup'))
-                        ) && (meta.size * meta.num_lanes == 1)
-                    ) { "mapped/${meta.id}/${it}" }
-                    else { null }
-                }
-            ]
+        withName: "(BWAMEM.*_MEM|DRAGMAP_ALIGN)" {
+            // Markduplicates Spark NEEDS name-sorted reads or runtime goes through the roof
+            // However if it's skipped, reads need to be coordinate-sorted
+            // Only name sort if Spark for Markduplicates + duplicate marking is not skipped
+            // Currently SENTIEON_BWAMEM only supports coordinate sorting the reads.
+            ext.args2        = { params.use_gatk_spark && params.use_gatk_spark.contains('markduplicates') && (!params.skip_tools || (params.skip_tools && !params.skip_tools.split(',').contains('markduplicates'))) ? '-n' : '' }
         }
 
         withName: "BWAMEM.*_MEM|SENTIEON_BWAMEM" {

conf/modules/markduplicates.config

@@ -33,6 +33,16 @@ process {
         ]
     }
 
+    withName: 'NFCORE_SAREK:SAREK:(BAM_MARKDUPLICATES|BAM_MARKDUPLICATES_SPARK):CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
+        ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
+        ext.prefix       = { "${meta.id}.md.cram" }
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/reports/samtools/${meta.id}" },
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+
     withName: 'BAM_TO_CRAM_MAPPING' {
         // Run only when mapping should be saved as CRAM or when no MD is done
         ext.when         = (params.save_mapped && !params.save_output_as_bam) ||

conf/modules/modules.config

@@ -40,26 +40,6 @@ process {
         ]
     }
 
-    withName: 'NFCORE_SAREK:SAREK:(BAM_MARKDUPLICATES|BAM_MARKDUPLICATES_SPARK):CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
-        ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
-        ext.prefix       = { "${meta.id}.md.cram" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reports/samtools/${meta.id}" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: 'NFCORE_SAREK:SAREK:BAM_SENTIEON_DEDUP:CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
-        ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
-        ext.prefix       = { "${meta.id}.dedup.cram" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reports/samtools/${meta.id}" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
     withName: 'NFCORE_SAREK:SAREK:CRAM_QC_NO_MD:SAMTOOLS_STATS' {
         ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
         ext.prefix       = { "${meta.id}.sorted.cram" }

conf/modules/mutect2.config

@@ -15,6 +15,7 @@
 
 process {
     if (params.tools && params.tools.split(',').contains('mutect2')) {
+
         withName: 'GATK4_MUTECT2' {
             ext.prefix       = { meta.num_intervals <= 1 ? "${meta.id}.mutect2" : "${meta.id}.mutect2.${intervals.simpleName}" }
             ext.when         = { params.tools && params.tools.split(',').contains('mutect2') }

conf/modules/post_variant_calling.config

@@ -15,7 +15,9 @@
 // Like, for instance, concatenating the unannotated, germline vcf-files
 
 process {
+
     withName: 'GERMLINE_VCFS_CONCAT'{
+        ext.when         =  params.concatenate_vcfs
         publishDir       = [
             //specify to avoid publishing, overwritten otherwise
             enabled: false
@@ -24,6 +26,7 @@ process {
 
     withName: 'GERMLINE_VCFS_CONCAT_SORT'{
         ext.prefix       = { "${meta.id}.germline" }
+        ext.when         =  params.concatenate_vcfs
         publishDir       = [
             mode: params.publish_dir_mode,
                 path: { "${params.outdir}/variant_calling/concat/${meta.id}/" }
@@ -32,10 +35,12 @@ process {
 
     withName: 'TABIX_EXT_VCF' {
         ext.prefix       = { "${input.baseName}" }
+        ext.when         =  params.concatenate_vcfs
     }
 
     withName: 'TABIX_GERMLINE_VCFS_CONCAT_SORT'{
         ext.prefix       = { "${meta.id}.germline" }
+        ext.when         =  params.concatenate_vcfs
         publishDir       = [
             mode: params.publish_dir_mode,
                 path: { "${params.outdir}/variant_calling/concat/${meta.id}/" }

conf/modules/prepare_cache.config

@@ -14,6 +14,7 @@
 // PREPARE_CACHE
 
 process {
+
     // SNPEFF
     withName: 'SNPEFF_DOWNLOAD' {
         ext.when         = { params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('merge')) }

conf/modules/sentieon_dedup.config

@@ -34,4 +34,16 @@ process {
         ]
     }
 
+    if (params.tools && params.tools.contains('sentieon_dedup')) {
+        withName: 'NFCORE_SAREK:SAREK:BAM_SENTIEON_DEDUP:CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
+            ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
+            ext.prefix       = { "${meta.id}.dedup.cram" }
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/reports/samtools/${meta.id}" },
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+            ]
+        }
+    }
+
 }

conf/modules/sentieon_haplotyper.config

@@ -45,14 +45,16 @@ process {
         ]
     }
 
-    withName: '.*BAM_VARIANT_CALLING_SENTIEON_HAPLOTYPER:VCF_VARIANT_FILTERING_GATK:FILTERVARIANTTRANCHES' {
-        ext.prefix       = {"${meta.id}.haplotyper"}
-        ext.args         = { "--info-key CNN_1D" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/sentieon_haplotyper/${meta.id}/"},
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
-        ]
+    if (params.tools && params.tools.contains('sentieon_haplotyper')) {
+        withName: '.*BAM_VARIANT_CALLING_SENTIEON_HAPLOTYPER:VCF_VARIANT_FILTERING_GATK:FILTERVARIANTTRANCHES' {
+            ext.prefix       = {"${meta.id}.haplotyper"}
+            ext.args         = { "--info-key CNN_1D" }
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/variant_calling/sentieon_haplotyper/${meta.id}/"},
+                pattern: "*{vcf.gz,vcf.gz.tbi}"
+            ]
+        }
     }
 
 }

conf/modules/sentieon_joint_germline.config

@@ -15,7 +15,7 @@
 
 process {
 
-    withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:SENTIEON_GVCFTYPER' {
+    withName: 'SENTIEON_GVCFTYPER' {
         ext.args         = { "--allow-old-rms-mapping-quality-annotation-data" }
         ext.prefix       = { meta.intervals_name }
         publishDir       = [
@@ -24,32 +24,32 @@ process {
     }
 
     if (params.tools && params.tools.contains('sentieon_haplotyper') && params.joint_germline) {
-        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON::BCFTOOLS_SORT' {
+        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:BCFTOOLS_SORT' {
             ext.prefix       = { vcf.baseName - ".vcf" + ".sort" }
             publishDir       = [
                 enabled: false
             ]
         }
-    }
 
-    withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_GENOTYPEGVCFS' {
-        ext.prefix       = "joint_germline"
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
-        ]
-    }
+        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_GENOTYPEGVCFS' {
+            ext.prefix       = "joint_germline"
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/" },
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
+                pattern: "*{vcf.gz,vcf.gz.tbi}"
+            ]
+        }
 
-    withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_VQSR' {
-        ext.prefix       = "joint_germline_recalibrated"
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/"},
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
-        ]
+        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_VQSR' {
+            ext.prefix       = "joint_germline_recalibrated"
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/"},
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
+                pattern: "*{vcf.gz,vcf.gz.tbi}"
+            ]
+        }
     }
 
     withName: 'SENTIEON_VARCAL_INDEL' {
@@ -78,5 +78,4 @@ process {
         ext.args      = '--sensitivity 99.9 --var_type SNP'
     }
 
-
 }

conf/modules/umi.config

@@ -62,6 +62,7 @@ process {
             enabled: false
         ]
     }
+
     withName: 'GROUPREADSBYUMI' {
         publishDir       = [
             [   path: { "${params.outdir}/reports/umi/" },

conf/test.config

@@ -46,6 +46,7 @@ params {
 }
 
 process {
+
     withName:'.*:FREEC_SOMATIC'{
         ext.args = {
             [
@@ -69,27 +70,26 @@ process {
         }
     }
 
-    if (params.tools && params.tools.split(',').contains('mutect2')) {
-        if (params.joint_mutect2) {
-            withName: 'MUTECT2_PAIRED' {
-                ext.args         = { params.ignore_soft_clipped_bases ?
-                                        "--dont-use-soft-clipped-bases true --f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" :
-                                        "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" }
-            }
+    if (params.joint_mutect2) {
+        withName: 'MUTECT2_PAIRED' {
+            ext.args         = { params.ignore_soft_clipped_bases ?
+                                    "--dont-use-soft-clipped-bases true --f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" :
+                                    "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" }
         }
-        else {
-            withName: '.*MUTECT2_PAIRED'{
-                //sample name from when the test data was generated
-                ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
-            }
+    }
+    else {
+        withName: 'MUTECT2_PAIRED'{
+            //sample name from when the test data was generated
+            ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
         }
     }
 
-    withName: '.*:FILTERVARIANTTRANCHES'{
+    withName: 'FILTERVARIANTTRANCHES'{
         ext.args = { "--info-key CNN_1D --indel-tranche 0" }
     }
 }
 
+
 // Enable container engines/virtualisation envs for CI testing
 // only works when specified with the profile ENV
 // otherwise tests can be done with the regular provided profiles

conf/test/cache.config

@@ -63,6 +63,7 @@ process {
         ext.sentieon_auth_data_base64 = secrets.SENTIEON_AUTH_DATA_BASE64
     }
 
+    // This must contain .* in order to properly overwrite the standard config in test cases
     withName:'.*:FREEC_SOMATIC'{
         ext.args = {
             [
@@ -86,14 +87,12 @@ process {
         }
     }
 
-    if (params.tools && params.tools.split(',').contains('mutect2')) {
-        withName: '.*MUTECT2_PAIRED'{
-            //sample name from when the test data was generated
-            ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
-        }
+    withName: 'MUTECT2_PAIRED'{
+        //sample name from when the test data was generated
+        ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
     }
 
-    withName: '.*:FILTERVARIANTTRANCHES'{
+    withName: 'FILTERVARIANTTRANCHES'{
         ext.args = { "--info-key CNN_1D --indel-tranche 0" }
     }
 }