nf-core · atrigila · Sep 17, 2024 · Sep 16, 2024 · Sep 17, 2024 · Sep 17, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [[#420]](https://github.com/nf-core/smrnaseq/pull/420) - Fix [mirTrace produces an error in test nextflex](https://github.com/nf-core/smrnaseq/issues/419) - Allow config mode to be used in mirtrace/qc
 - [[#425]](https://github.com/nf-core/smrnaseq/pull/425) - Raise [minimum required NXF version for pipeline](https://github.com/nf-core/smrnaseq/issues/424) - usage of `arity` in some modules now requires this
 - [[#426]](https://github.com/nf-core/smrnaseq/pull/426) - Add [nf-core mirtop](https://github.com/nf-core/smrnaseq/issues/426) - replace local for nf-core `mirtop`
+- [[#429]](https://github.com/nf-core/smrnaseq/pull/429) - Make [saving of intermediate files optional](https://github.com/nf-core/smrnaseq/issues/424) - Allows user to choose whether to save intermediate files or not. Replaces several params that referred to the same such as `params.save_aligned` and `params.save_aligned_mirna_quant`.
 
 ## v2.3.1 - 2024-04-18 - Gray Zinc Dalmation Patch
 

diff --git a/conf/modules.config b/conf/modules.config
@@ -42,6 +42,14 @@ process {
         ]
     }
 
+    withName: 'UNTAR' {
+        publishDir = [
+            mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+
     //
     // FASTQ_FASTQC_UMITOOLS_FASTP
     //
@@ -110,6 +118,7 @@ process {
         publishDir = [
             path: { "${params.outdir}/fastqc/raw" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
@@ -119,6 +128,7 @@ process {
         publishDir = [
             path: { "${params.outdir}/fastqc/trimmed" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
@@ -161,6 +171,7 @@ process {
         publishDir = [
             path: { "${params.outdir}/bowtie_index/genome" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
@@ -249,27 +260,31 @@ process {
         publishDir = [
             path: { "${params.outdir}/mirna_quant/reference" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
     withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:PARSE_.*' {
         publishDir = [
             path: { "${params.outdir}/mirna_quant/reference" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
     withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:INDEX_MATURE' {
         publishDir = [
             path: { "${params.outdir}/bowtie_index/mirna_mature" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
     withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:INDEX_HAIRPIN' {
         publishDir = [
             path: { "${params.outdir}/bowtie_index/mirna_hairpin" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
@@ -278,14 +293,15 @@ process {
             path: { "${params.outdir}/mirna_quant/bam/mature" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: params.save_aligned_mirna_quant
+            enabled: params.save_intermediates
         ]
     }
     withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:BAM_STATS_MATURE:.*' {
         ext.prefix = { "${meta.id}.sorted" }
         publishDir = [
             path: { "${params.outdir}/mirna_quant/bam/mature" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
@@ -294,14 +310,15 @@ process {
             path: { "${params.outdir}/mirna_quant/bam/hairpin" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: params.save_aligned_mirna_quant
+            enabled: params.save_intermediates
         ]
     }
     withName: 'NFCORE_SMRNASEQ:MIRNA_QUANT:BAM_STATS_HAIRPIN:.*' {
         ext.prefix = { "${meta.id}.sorted" }
         publishDir = [
             path: { "${params.outdir}/mirna_quant/bam/hairpin" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
@@ -316,6 +333,7 @@ process {
         publishDir = [
             path: { "${params.outdir}/mirna_quant/seqcluster" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
         ext.args    = "-m 1 --min_size 15"
@@ -326,7 +344,7 @@ process {
             path: { "${params.outdir}/mirna_quant/bam/seqcluster" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: params.save_aligned_mirna_quant
+            enabled: params.save_intermediates
         ]
     }
     withName: 'MIRTOP_QUANT' {
@@ -378,12 +396,16 @@ process {
         publishDir = [
             path: { "${params.outdir}/genome_quant/bam" },
             mode: params.publish_dir_mode,
+            enabled: params.save_intermediates,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
     withName: 'SAMTOOLS_INDEX' {
         ext.args = '-c'
+        publishDir = [
+            enabled: params.save_intermediates,
+        ]
     }
 
     withName: 'NFCORE_SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' {
@@ -399,7 +421,7 @@ process {
             path: { "${params.outdir}/genome_quant/bam" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: params.save_aligned
+            enabled: params.save_intermediates
         ]
     }
 

diff --git a/docs/output.md b/docs/output.md
@@ -27,6 +27,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
 - [MultiQC](#multiqc) - aggregate report, describing results of the whole pipeline
 - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution
 
+If `--save_intermediates` is specified, intermediate files generated by each process will be saved in the output directory.
+
 ### FastQC
 
 <details markdown="1">
@@ -102,6 +104,8 @@ MultiQC reports the number of reads that were removed by each of the contaminant
 - `sample_mature_hairpin_unmapped.fq.gz`: Unmapped reads against miRNA precursors (hairpins)
 - `sample_mature_hairpin_genome.bam`: The aligned BAM file of alignment against that didn't map to the precursor.
 
+If `--save_intermediates` is specified, these files will be placed in this directory.
+
 ## SAMtools
 
 [SAMtools](http://samtools.sourceforge.net/) is used for sorting and indexing the output BAM files from Bowtie. In addition, the numbers of features are counted with the `idxstats` option.

diff --git a/docs/usage.md b/docs/usage.md
@@ -164,6 +164,10 @@ genome: 'GRCh37'
 
 You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch).
 
+## Optional parameters
+
+If `--save_intermediates` is specified, the intermediate files generated in the pipeline will be saved in the output directory.
+
 ### Updating the pipeline
 
 When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline:

diff --git a/nextflow.config b/nextflow.config
@@ -10,7 +10,6 @@
 params {
 
     // Input options
-
     input                       = null
 
     // Protocol default (override via config profile - NOT directly via parameter!)
@@ -28,10 +27,11 @@ params {
     mirgenedb_hairpin           = null
     mirgenedb_gff               = null
     mirgenedb_species           = null
-    save_aligned                = false
-    save_aligned_mirna_quant    = true
     bowtie_index                = null
 
+    // General pipeline configuration
+    save_intermediates          = false
+
     // UMI handling
     with_umi                      = false
     // skips umitools extract in FASTQ_FASTQC_UMITOOLS_FASTP subworkflow. Needs to be true for fastq mode in collapsing reads
@@ -50,7 +50,7 @@ params {
     fastp_min_length            = 17
     fastp_known_mirna_adapters  = "$projectDir/assets/known_adapters.fa"
     save_trimmed_fail           = false
-    save_merged                 = true
+    save_merged                 = true                  // All boolean parameters should evaluate to false as per nf-core guidelines, should we change this?
     skip_fastqc                 = false
     skip_multiqc                = false
     skip_mirdeep                = false

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -47,6 +47,11 @@
                     "default": "custom",
                     "fa_icon": "fas fa-atom",
                     "enum": ["custom", "illumina", "nextflex", "qiaseq", "cats"]
+                },
+                "save_intermediates": {
+                    "type": "boolean",
+                    "description": "Save all intermediate files (e.g. fastq, bams) to output directory",
+                    "fa_icon": "fas fa-save"
                 }
             }
         },
@@ -185,19 +190,6 @@
                     "help_text": "Saving generated references means that you can use them for future pipeline runs, reducing processing times.",
                     "fa_icon": "fas fa-save"
                 },
-                "save_aligned": {
-                    "type": "boolean",
-                    "fa_icon": "fas fa-save",
-                    "help_text": "Save aligned reads of initial bowtie mapping.",
-                    "description": "Save aligned reads of initial mapping in bam format."
-                },
-                "save_aligned_mirna_quant": {
-                    "type": "boolean",
-                    "fa_icon": "fas fa-save",
-                    "default": true,
-                    "help_text": "Save aligned reads of the bowtie runs in BOWTIE_MAP_MATURE, BOWTIE_MAP_HAIRPIN, and BOWTIE_MAP_SEQCLUSTER.",
-                    "description": "Save aligned reads of miRNA quant subworkflow in bam format."
-                },
                 "igenomes_ignore": {
                     "type": "boolean",
                     "description": "Do not load the iGenomes reference config.",