From 0b7a029584c7b8e69a73b5beb9d145d64f64d7a5 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Thu, 28 Mar 2024 16:44:17 +0100 Subject: [PATCH 1/3] :wrench: update MULTIQC config --- .vscode/settings.json | 1 + assets/methods_description_template.yml | 29 ++++++++++++++++ assets/multiqc_config.yml | 22 +++++++++++-- nextflow.config | 2 +- workflows/resequencing-mem.nf | 44 ++++++++++++++++--------- 5 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 assets/methods_description_template.yml diff --git a/.vscode/settings.json b/.vscode/settings.json index 49981a8..1b66a96 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,6 +13,7 @@ "downsampling", "fasta", "fastq", + "flagstat", "freebayes", "ibba", "markduplicate", diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml new file mode 100644 index 0000000..1aeec64 --- /dev/null +++ b/assets/methods_description_template.yml @@ -0,0 +1,29 @@ +id: "nf-resequencing-mem-methods-description" +description: "Suggested text and references to use when describing pipeline usage within the methods section of a publication." +section_name: "cnr-ibba/nf-resequencing-mem Methods Description" +section_href: "https://github.com/cnr-ibba/nf-resequencing-mem" +plot_type: "html" +## TODO nf-core: Update the HTML below to your preferred methods description, e.g. add publication citation for this pipeline +## You inject any metadata in the Nextflow '${workflow}' object +data: | +

Methods

+

Data was processed using cnr-ibba/nf-resequencing-mem v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020), utilising reproducible software environments from the Bioconda (Grüning et al., 2018) and Biocontainers (da Veiga Leprevost et al., 2017) projects.

+

The pipeline was executed with Nextflow v${workflow.nextflow.version} (Di Tommaso et al., 2017) with the following command:

+ 
${workflow.commandLine}
+

${tool_citations}

+

References

+ +
+
Notes:
+ +
diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 6a8c152..63ed6a4 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -3,13 +3,31 @@ report_comment: > analysis pipeline. For information about how to interpret these results, please see the documentation. report_section_order: - software_versions: + "nf-resequencing-mem-methods-description": order: -1000 - cnr-ibba/nf-resequencing-mem-summary: + software_versions: order: -1001 + cnr-ibba/nf-resequencing-mem-summary: + order: -1002 export_plots: true custom_logo: "cnr-ibba.png" custom_logo_url: "https://ibba.cnr.it/en/" custom_logo_title: "CNR-IBBA" + +extra_fn_clean_exts: + - ".cram" + - type: remove + pattern: _T1 + +top_modules: + - fastqc + +module_order: + - fastqc + - cutadapt + - samtools + - picard + - bcftools + - snpeff diff --git a/nextflow.config b/nextflow.config index 799dc73..674cbf8 100644 --- a/nextflow.config +++ b/nextflow.config @@ -17,7 +17,7 @@ params { genome_bwa_index = null // MultiQC options - multiqc_config = "$baseDir/assets/multiqc_config.yml" + multiqc_config = null multiqc_logo = "$baseDir/assets/cnr-ibba.png" multiqc_title = null max_multiqc_email_size = '25.MB' diff --git a/workflows/resequencing-mem.nf b/workflows/resequencing-mem.nf index e88762e..93fe5ea 100644 --- a/workflows/resequencing-mem.nf +++ b/workflows/resequencing-mem.nf @@ -213,25 +213,39 @@ workflow RESEQUENCING_MEM { } // get only the data I need for a MultiQC step - multiqc_input = FASTQC.out.html.map{it[1]}.ifEmpty([]) - .concat(FASTQC.out.zip.map{it[1]}.ifEmpty([])) - .concat(TRIMGALORE.out.log.map{it[1]}.ifEmpty([])) - .concat(CRAM_MARKDUPLICATES_PICARD.out.metrics.map{it[1]}.ifEmpty([])) - .concat(CRAM_MARKDUPLICATES_PICARD.out.stats.map{it[1]}.ifEmpty([])) - .concat(CRAM_MARKDUPLICATES_PICARD.out.idxstats.map{it[1]}.ifEmpty([])) - .concat(CRAM_MARKDUPLICATES_PICARD.out.flagstat.map{it[1]}.ifEmpty([])) - .concat(CRAM_MARKDUPLICATES_PICARD.out.stats.map{it[1]}.ifEmpty([])) - .concat(snpeff_report.map{it[1]}.ifEmpty([])) - .collect() - // .view() + ch_multiqc_files = FASTQC.out.html.map{it[1]}.ifEmpty([]) + .concat(FASTQC.out.zip.map{it[1]}.ifEmpty([])) + .concat(TRIMGALORE.out.log.map{it[1]}.ifEmpty([])) + .concat(CRAM_MARKDUPLICATES_PICARD.out.metrics.map{it[1]}.ifEmpty([])) + .concat(CRAM_MARKDUPLICATES_PICARD.out.stats.map{it[1]}.ifEmpty([])) + .concat(CRAM_MARKDUPLICATES_PICARD.out.idxstats.map{it[1]}.ifEmpty([])) + .concat(CRAM_MARKDUPLICATES_PICARD.out.flagstat.map{it[1]}.ifEmpty([])) + .concat(CRAM_MARKDUPLICATES_PICARD.out.coverage.map{it[1]}.ifEmpty([])) + .concat(BCFTOOLS_STATS.out.stats.map{it[1]}.ifEmpty([])) + .concat(snpeff_report.map{it[1]}.ifEmpty([])) + // .view() // prepare multiqc_config file - multiqc_config = Channel.fromPath(params.multiqc_config) - multiqc_logo = Channel.fromPath(params.multiqc_logo) + ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + + // TODO: requires nf-validation plugin and other stuff + // summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + // ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + // ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + // ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) + // ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + // ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + // ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) // calling MultiQC - MULTIQC(multiqc_input, multiqc_config, [], multiqc_logo) - ch_versions = ch_versions.mix(MULTIQC.out.versions) + MULTIQC ( + ch_multiqc_files.collect(), + ch_multiqc_config.toList(), + ch_multiqc_custom_config.toList(), + ch_multiqc_logo.toList() + ) // return software version CUSTOM_DUMPSOFTWAREVERSIONS ( From a6cc22858cfd3ed07a5a34087e41c5705cf15454 Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Thu, 28 Mar 2024 16:54:02 +0100 Subject: [PATCH 2/3] :recycle: minor fixes :recycle: minor fixes --- assets/multiqc_config.yml | 2 +- conf/modules.config | 8 ++++++++ workflows/resequencing-mem.nf | 14 +++++++------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 63ed6a4..f89f6c3 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -7,7 +7,7 @@ report_section_order: order: -1000 software_versions: order: -1001 - cnr-ibba/nf-resequencing-mem-summary: + cnr-ibba-nf-resequencing-mem-summary: order: -1002 export_plots: true diff --git a/conf/modules.config b/conf/modules.config index 1487890..c2c0a4f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -139,6 +139,14 @@ process { ] } + withName: SAMTOOLS_DEPTH { + publishDir = [ + path: { "${params.outdir}/coverage" }, + mode: params.publish_dir_mode, + pattern: "*.gz" + ] + } + withName: FREEBAYES_SPLITCRAM { publishDir = [ enabled: false diff --git a/workflows/resequencing-mem.nf b/workflows/resequencing-mem.nf index 93fe5ea..f839273 100644 --- a/workflows/resequencing-mem.nf +++ b/workflows/resequencing-mem.nf @@ -64,13 +64,13 @@ workflow RESEQUENCING_MEM { ) .reads .map { - meta, fastq -> - def meta_clone = meta.clone() - tmp = meta_clone.id.split('_') - if (tmp.size() > 1) { - meta_clone.id = tmp[0..-2].join('_') - } - [ meta_clone, fastq ] + meta, fastq -> + def meta_clone = meta.clone() + tmp = meta_clone.id.split('_') + if (tmp.size() > 1) { + meta_clone.id = tmp[0..-2].join('_') + } + [ meta_clone, fastq ] } .groupTuple(by: [0]) .branch { From 53c034cb7c54d64eded497d28f4333b3e0ee904d Mon Sep 17 00:00:00 2001 From: Paolo Cozzi Date: Thu, 28 Mar 2024 16:46:58 +0100 Subject: [PATCH 3/3] :arrow_up: upgrade MULTIQC module --- modules.json | 2 +- modules/nf-core/multiqc/environment.yml | 2 +- modules/nf-core/multiqc/main.nf | 6 +-- modules/nf-core/multiqc/meta.yml | 1 - modules/nf-core/multiqc/tests/main.nf.test | 53 +++++++++++++------ .../nf-core/multiqc/tests/main.nf.test.snap | 41 ++++++++++++++ 6 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 modules/nf-core/multiqc/tests/main.nf.test.snap diff --git a/modules.json b/modules.json index f646dcd..964e6d8 100644 --- a/modules.json +++ b/modules.json @@ -64,7 +64,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "4ab13872435962dadc239979554d13709e20bf29", + "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", "installed_by": ["modules"] }, "picard/markduplicates": { diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index bc0bdb5..ca39fb6 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.18 + - bioconda::multiqc=1.21 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 00cc48d..47ac352 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.18--pyhdfd78af_0' : - 'biocontainers/multiqc:1.18--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : + 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" @@ -43,7 +43,7 @@ process MULTIQC { stub: """ - touch multiqc_data + mkdir multiqc_data touch multiqc_plots touch multiqc_report.html diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index f1aa660..45a9bc3 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,4 +1,3 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: multiqc description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index c2dad21..f1c4242 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -3,19 +3,17 @@ nextflow_process { name "Test Process MULTIQC" script "../main.nf" process "MULTIQC" + tag "modules" tag "modules_nfcore" tag "multiqc" - test("MULTIQC: FASTQC") { + test("sarscov2 single-end [fastqc]") { when { - params { - outdir = "$outputDir" - } process { """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) input[1] = [] input[2] = [] input[3] = [] @@ -26,23 +24,20 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert path(process.out.report.get(0)).exists() }, - { assert path(process.out.data.get(0)).exists() }, - { assert path(process.out.versions.get(0)).getText().contains("multiqc") } + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("multiqc_versions_single") } ) } } - test("MULTIQC: FASTQC and a config file") { + test("sarscov2 single-end [fastqc] [config]") { when { - params { - outdir = "$outputDir" - } process { """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] @@ -53,9 +48,35 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert path(process.out.report.get(0)).exists() }, - { assert path(process.out.data.get(0)).exists() }, - { assert path(process.out.versions.get(0)).getText().contains("multiqc") } + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("multiqc_versions_config") } + ) + } + } + + test("sarscov2 single-end [fastqc] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.report.collect { file(it).getName() } + + process.out.data.collect { file(it).getName() } + + process.out.plots.collect { file(it).getName() } + + process.out.versions ).match("multiqc_stub") } ) } diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap new file mode 100644 index 0000000..bfebd80 --- /dev/null +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -0,0 +1,41 @@ +{ + "multiqc_versions_single": { + "content": [ + [ + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-29T08:48:55.657331" + }, + "multiqc_stub": { + "content": [ + [ + "multiqc_report.html", + "multiqc_data", + "multiqc_plots", + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-29T08:49:49.071937" + }, + "multiqc_versions_config": { + "content": [ + [ + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-29T08:49:25.457567" + } +} \ No newline at end of file