Fix intervals

CHANGELOG.md

@@ -64,6 +64,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#463](https://github.com/nf-core/sarek/pull/463), [#468](https://github.com/nf-core/sarek/pull/468) - Fix `nf-core lint`
 - [#513](https://github.com/nf-core/sarek/pull/513), [#527](https://github.com/nf-core/sarek/pull/527) - CNV is back
 - [#529](https://github.com/nf-core/sarek/pull/529) - Do not save `versions.yml` files
+- [#524](https://github.com/nf-core/sarek/pull/524) - Fix intervals usage by counting the actual list of scatter/gather files produced and not overall number of intervals
 
 ### Deprecated
 

conf/modules.config

@@ -298,8 +298,8 @@ process {
         ext.prefix       = { "${meta.id}.md" }
         publishDir       = [
             mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reports/${meta.id}/markduplicates" },
-            pattern: "metrics"
+            path: { "${params.outdir}/reports/markduplicates/${meta.id}" },
+            pattern: "*metrics"
         ]
     }
 
@@ -344,34 +344,41 @@ process {
 
 // PREPARE_RECALIBRATION
 
-    withName: 'BASERECALIBRATOR|BASERECALIBRATOR_SPARK|GATHERBQSRREPORTS' {
+    withName: 'BASERECALIBRATOR|BASERECALIBRATOR_SPARK' {
         ext.prefix       = {"${meta.id}.recal"}
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/preprocessing/${meta.id}/recal_table" },
-            pattern: "*table"
+            pattern: "*table",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
 
     withName: 'GATHERBQSRREPORTS' {
-        ext.when         = { !params.no_intervals }
+        ext.prefix       = {"${meta.id}.recal"}
+        ext.when         = { meta.num_intervals > 1 }
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/preprocessing/${meta.id}/recal_table" },
+            pattern: "*table",
+        ]
     }
 
 // RECALIBRATE
 
     withName: 'APPLYBQSR|APPLYBQSR_SPARK' {
         ext.prefix       = {"${meta.id}.recal"}
         publishDir       = [
-            enabled: !params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/preprocessing/${meta.id}/recalibrated" },
-            pattern: "*cram"
+            pattern: "*cram",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
 
     withName: 'NFCORE_SAREK:SAREK:(RECALIBRATE|RECALIBRATE_SPARK):MERGE_INDEX_CRAM:MERGE_CRAM' {
         ext.prefix       = { "${meta.id}.recal" }
-        ext.when         = { !params.no_intervals }
+        ext.when         = { meta.num_intervals > 1 }
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/preprocessing/${meta.id}/recalibrated" },
@@ -473,7 +480,6 @@ process{
     // DEEPVARIANT
     withName: 'CONCAT_DEEPVARIANT_.*' {
         publishDir       = [
-            enabled: !params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/deepvariant" },
             pattern: "*{vcf.gz,vcf.gz.tbi}"
@@ -486,24 +492,23 @@ process{
         ext.args         = { params.wes ?  "--model_type WES" : "--model_type WGS" }
         ext.when         = { params.tools && params.tools.contains('deepvariant') }
         publishDir       = [
-            enabled: params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/deepvariant" },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
+            pattern: "*vcf.gz",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
-    withName : 'TABIX_VC_DEEPVARIANT.*' {
+    withName : 'TABIX_VC_DEEPVARIANT_.*' {
         publishDir       = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/deepvariant" },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
+            pattern: "*tbi"
         ]
     }
 
     // FREEBAYES
     withName: 'CONCAT_FREEBAYES' {
         publishDir       = [
-            enabled: !params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/freebayes" },
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
@@ -513,10 +518,10 @@ process{
         ext.args         = '--min-alternate-fraction 0.1 --min-mapping-quality 1'
         ext.when         = { params.tools && params.tools.contains('freebayes') }
         publishDir       = [
-            enabled: params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/freebayes" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+            pattern: "*vcf.gz",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
     withName : 'TABIX_VC_FREEBAYES' {
@@ -561,7 +566,6 @@ process{
     // MANTA
     withName: 'CONCAT_MANTA.*' {
         publishDir       = [
-            enabled: !params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/manta" },
             pattern: "*{vcf.gz,vcf.gz.tbi}"
@@ -580,17 +584,16 @@ process{
         ext.args         = { params.wes ? "--exome" : "" }
         ext.when         = { params.tools && params.tools.contains('manta') }
         publishDir       = [
-            enabled: params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/manta" },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
+            pattern: "*{vcf.gz,vcf.gz.tbi}",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
 
     // STRELKA
     withName: 'CONCAT_STRELKA.*' {
         publishDir       = [
-            enabled: !params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/strelka" },
             pattern: "*{vcf.gz,vcf.gz.tbi}"
@@ -606,10 +609,10 @@ process{
         ext.args         = { params.wes ? "--exome" : "" }
         ext.when         = { params.tools && params.tools.contains('strelka') }
         publishDir       = [
-            enabled: params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/strelka" },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
+            pattern: "*{vcf.gz,vcf.gz.tbi}",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
 
@@ -632,9 +635,8 @@ process{
     }
 
     withName: 'CAT_MPILEUP_.*' {
-        ext.when =  { !params.no_intervals }
+        ext.when =  { meta.num_intervals > 1 }
         publishDir       = [
-            enabled: !params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/controlfreec" },
             pattern: "*mpileup"
@@ -712,10 +714,10 @@ process{
     withName: 'MPILEUP_.*' {
         ext.when         = { params.tools && params.tools.contains('controlfreec') }
         publishDir       = [
-            enabled: params.no_intervals,
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/variant_calling/${meta.id}/controlfreec" },
-            pattern: "*mpileup"
+            pattern: "*mpileup",
+            saveAs: { meta.num_intervals > 1 ? null : it }
         ]
     }
 
@@ -959,11 +961,22 @@ process {
     withName: 'ENSEMBLVEP' {
         ext.args          = '--everything --filter_common --per_gene --total_length --offline'
         container         = { "nfcore/vep:104.3.${params.genome}" }
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/reports/EnsemblVEP/${meta.id}/${meta.variantcaller}" },
+            pattern: "*html"
+        ]
     }
 
     withName: 'SNPEFF' {
         ext.args          = '-nodownload -canon -v'
         container         = { "nfcore/snpeff:5.0.${params.genome}" }
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/reports/SnpEff/${meta.id}/${meta.variantcaller}" },
+            pattern: "*csv",
+            saveAs: { params.tools.contains('snpeff') ? it : null }
+        ]
     }
 
     withName: 'ANNOTATION_BGZIPTABIX' {
@@ -979,6 +992,12 @@ if (params.tools && (params.tools.contains('snpeff') || params.tools.contains('m
     process {
         withName: 'NFCORE_SAREK:SAREK:ANNOTATE:ANNOTATION_SNPEFF:ANNOTATION_BGZIPTABIX' {
             ext.prefix       = {"${meta.id}_snpEff.ann.vcf"}
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/annotation/${meta.id}/${meta.variantcaller}" },
+                pattern: "*{gz,gz.tbi}",
+                saveAs: { params.tools.contains('snpeff') ? it : null }
+            ]
         }
     }
 }

modules.json

@@ -25,7 +25,7 @@
                 "git_sha": "eeda4136c096688d04cc40bb3c70d948213ed641"
             },
             "cat/fastq": {
-                "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"
+                "git_sha": "9aadd9a6d3f5964476582319b3a1c54a3e3fe7c9"
             },
             "cnvkit/batch": {
                 "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"
@@ -121,7 +121,7 @@
                 "git_sha": "409af2f27cbe45109acc7fee70718d2bf20aa449"
             },
             "gatk4/haplotypecaller": {
-                "git_sha": "409af2f27cbe45109acc7fee70718d2bf20aa449"
+                "git_sha": "68f1c27169946f931ea4318911de5681f88b2961"
             },
             "gatk4/intervallisttobed": {
                 "git_sha": "409af2f27cbe45109acc7fee70718d2bf20aa449"
@@ -130,7 +130,7 @@
                 "git_sha": "409af2f27cbe45109acc7fee70718d2bf20aa449"
             },
             "gatk4/markduplicates": {
-                "git_sha": "409af2f27cbe45109acc7fee70718d2bf20aa449"
+                "git_sha": "0511e7fbbfa4ba41940d33b687b1cc90227b4eb8"
             },
             "gatk4/markduplicatesspark": {
                 "git_sha": "e04970b7d249365cafa5a52912f9a28840481c05"
@@ -217,16 +217,16 @@
                 "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe"
             },
             "tabix/tabix": {
-                "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe"
+                "git_sha": "b3e9b88e80880f450ad79a95b2b7aa05e1de5484"
             },
             "tiddit/sv": {
-                "git_sha": "fd5f6f5f4ffef4ab5a4e809bd3211bbc71c38d30"
+                "git_sha": "57cb730e78634673fb254a77606e014ce942734c"
             },
             "trimgalore": {
-                "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"
+                "git_sha": "85ec13ff1fc2196c5a507ea497de468101baabed"
             },
             "untar": {
-                "git_sha": "9ae34a01d1747019fd37753ff4cafb05aec35a2b"
+                "git_sha": "9aadd9a6d3f5964476582319b3a1c54a3e3fe7c9"
             },
             "vcftools": {
                 "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d"