diff --git a/README.md b/README.md index 50e94346..ccb136f0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![GitHub Actions CI Status](https://github.com/nf-core/smrnaseq/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/ci.yml) -[![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.10696391?labelColor=000000)](https://doi.org/10.5281/zenodo.10696391) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) diff --git a/modules.json b/modules.json index 1d31a680..109997b3 100644 --- a/modules.json +++ b/modules.json @@ -17,12 +17,12 @@ }, "fastp": { "branch": "master", - "git_sha": "003920c7f9a8ae19b69a97171922880220bedf56", + "git_sha": "95cf5fe0194c7bf5cb0e3027a2eb7e7c89385080", "installed_by": ["fastq_fastqc_umitools_fastp", "modules"] }, "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["fastq_fastqc_umitools_fastp"] }, "multiqc": { @@ -57,12 +57,12 @@ }, "umicollapse": { "branch": "master", - "git_sha": "ba3f3df395d2719dcef5c67189042a1dc555c701", + "git_sha": "b97197968ac12dde2463fa54541f6350c46f2035", "installed_by": ["modules"] }, "umitools/extract": { "branch": "master", - "git_sha": "ff7e93715a2acecf3f143ec78c9858deba2984d0", + "git_sha": "d2c5e76f291379f3dd403e48e46ed7e6ba5da744", "installed_by": ["fastq_fastqc_umitools_fastp", "modules"] }, "untarfiles": { @@ -86,7 +86,7 @@ }, "fastq_fastqc_umitools_fastp": { "branch": "master", - "git_sha": "003920c7f9a8ae19b69a97171922880220bedf56", + "git_sha": "cabcc0dadf8366aa7a9930066a7b3dd90d9825d5", "installed_by": ["subworkflows"] }, "utils_nextflow_pipeline": { diff --git a/modules/nf-core/fastp/main.nf b/modules/nf-core/fastp/main.nf index 2a3b679e..4fc19b74 100644 --- a/modules/nf-core/fastp/main.nf +++ b/modules/nf-core/fastp/main.nf @@ -29,7 +29,7 @@ process FASTP { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def adapter_list = adapter_fasta ? "--adapter_fasta ${adapter_fasta}" : "" - def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' + def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--failed_out ${prefix}.paired.fail.fastq.gz --unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' // Added soft-links to original fastqs for consistent naming in MultiQC // Use single ended for interleaved. Add --interleaved_in in config. if ( task.ext.args?.contains('--interleaved_in') ) { diff --git a/modules/nf-core/fastp/tests/main.nf.test b/modules/nf-core/fastp/tests/main.nf.test index 9b3f9a38..6f1f4897 100644 --- a/modules/nf-core/fastp/tests/main.nf.test +++ b/modules/nf-core/fastp/tests/main.nf.test @@ -251,7 +251,8 @@ nextflow_process { } test("fastp test_fastp_interleaved") { - config './nextflow.config' + + config './nextflow.interleaved.config' when { params { outdir = "$outputDir" @@ -277,7 +278,7 @@ nextflow_process { def html_text = [ "Q20 bases:25.719000 K (93.033098%)", "paired end (151 cycles + 151 cycles)"] def log_text = [ "Q20 bases: 12922(92.9841%)", - "reads passed filter: 198"] + "reads passed filter: 162"] def read_lines = [ "@ERR5069949.2151832 NS500628:121:HK3MMAFX2:2:21208:10793:15304/1", "TCATAAACCAAAGCACTCACAGTGTCAACAATTTCAGCAGGACAACGCCGACAAGTTCCGAGGAACATGTCTGGACCTATAGTTTTCATAAGTCTACACACTGAATTGAAATATTCTGGTTCTAGTGTGCCCTTAGTTAGCAATGTGCGT", "AAAAAAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAEEEEE - { assert path(process.out.reads_fail.get(0).get(1).get(1)).linesGzip.contains(failed_read2_line) } + { assert path(process.out.reads_fail.get(0).get(1).get(2)).linesGzip.contains(failed_read2_line) } } }, { html_text.each { html_part -> diff --git a/modules/nf-core/fastp/tests/main.nf.test.snap b/modules/nf-core/fastp/tests/main.nf.test.snap index b4c0e1dd..3e876288 100644 --- a/modules/nf-core/fastp/tests/main.nf.test.snap +++ b/modules/nf-core/fastp/tests/main.nf.test.snap @@ -7,7 +7,7 @@ "id": "test", "single_end": true }, - "test.fastp.json:md5,168f516f7bd4b7b6c32da7cba87299a4" + "test.fastp.json:md5,b24e0624df5cc0b11cd5ba21b726fb22" ] ] ], @@ -15,7 +15,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-17T18:08:06.123035" + "timestamp": "2024-03-18T16:19:15.063001" }, "test_fastp_paired_end_merged-for_stub_match": { "content": [ @@ -65,7 +65,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-17T18:06:00.223817" + "timestamp": "2024-03-18T16:18:43.526412" }, "versions_paired_end": { "content": [ @@ -112,7 +112,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-01T12:03:37.827323085" + "timestamp": "2024-03-18T16:19:15.111894" }, "test_fastp_paired_end_merged_match": { "content": [ @@ -283,7 +283,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-01T11:57:30.791982648" + "timestamp": "2024-03-18T16:18:43.580336" }, "versions_paired_end_merged_adapterlist": { "content": [ diff --git a/modules/nf-core/fastp/tests/nextflow.interleaved.config b/modules/nf-core/fastp/tests/nextflow.interleaved.config new file mode 100644 index 00000000..4be8dbd2 --- /dev/null +++ b/modules/nf-core/fastp/tests/nextflow.interleaved.config @@ -0,0 +1,5 @@ +process { + withName: FASTP { + ext.args = "--interleaved_in -e 30" + } +} diff --git a/modules/nf-core/fastp/tests/nextflow.config b/modules/nf-core/fastp/tests/nextflow.save_failed.config similarity index 50% rename from modules/nf-core/fastp/tests/nextflow.config rename to modules/nf-core/fastp/tests/nextflow.save_failed.config index 0f7849ad..53b61b0c 100644 --- a/modules/nf-core/fastp/tests/nextflow.config +++ b/modules/nf-core/fastp/tests/nextflow.save_failed.config @@ -1,6 +1,5 @@ process { - withName: FASTP { - ext.args = "--interleaved_in" + ext.args = "-e 30" } } diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 9e19a74c..d79f1c86 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/umicollapse/tests/main.nf.test b/modules/nf-core/umicollapse/tests/main.nf.test index 36cd748b..2dec45b2 100644 --- a/modules/nf-core/umicollapse/tests/main.nf.test +++ b/modules/nf-core/umicollapse/tests/main.nf.test @@ -34,10 +34,7 @@ nextflow_process { script "../../bwa/index/main.nf" process{ """ - input[0] = [ - [ id:'sarscov2'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] + input[0] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] """ } } @@ -47,7 +44,8 @@ nextflow_process { """ input[0] = UMITOOLS_EXTRACT.out.reads input[1] = BWA_INDEX.out.index - input[2] = true + input[2] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = true """ } } @@ -117,7 +115,8 @@ nextflow_process { """ input[0] = UMITOOLS_EXTRACT.out.reads input[1] = BWA_INDEX.out.index - input[2] = true + input[2] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = true """ } } @@ -214,7 +213,8 @@ nextflow_process { """ input[0] = UMITOOLS_EXTRACT.out.reads input[1] = BWA_INDEX.out.index - input[2] = true + input[2] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = true """ } } diff --git a/modules/nf-core/umicollapse/tests/main.nf.test.snap b/modules/nf-core/umicollapse/tests/main.nf.test.snap index 23916f0b..861e9ca6 100644 --- a/modules/nf-core/umicollapse/tests/main.nf.test.snap +++ b/modules/nf-core/umicollapse/tests/main.nf.test.snap @@ -16,9 +16,9 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T11:24:31.850566925" + "timestamp": "2024-03-14T13:41:23.869211282" }, "umicollapse fastq tests": { "content": [ @@ -117,8 +117,8 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T11:24:44.166029769" + "timestamp": "2024-03-14T13:41:54.486079388" } } \ No newline at end of file diff --git a/modules/nf-core/umitools/extract/tests/main.nf.test b/modules/nf-core/umitools/extract/tests/main.nf.test index 83a77a1b..2a8eba15 100644 --- a/modules/nf-core/umitools/extract/tests/main.nf.test +++ b/modules/nf-core/umitools/extract/tests/main.nf.test @@ -12,9 +12,6 @@ nextflow_process { test("Should run without failures") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ [ id:'test', single_end:true ], // meta map @@ -26,10 +23,9 @@ nextflow_process { then { assertAll ( - { assert process.success }, - { assert snapshot(process.out.versions).match("versions") } + { assert process.success }, + { assert snapshot(process.out.versions).match("versions") } ) } - } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test index 8f1d82d3..961b5b4f 100644 --- a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test @@ -3,6 +3,8 @@ nextflow_workflow { name "Test Workflow FASTQ_FASTQC_UMITOOLS_FASTP" script "../main.nf" workflow "FASTQ_FASTQC_UMITOOLS_FASTP" + config './nextflow.config' + tag "subworkflows" tag "subworkflows_nfcore" tag "subworkflows/fastq_fastqc_umitools_fastp" @@ -29,8 +31,10 @@ nextflow_workflow { input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true)] + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] ]) input[1] = skip_fastqc input[2] = with_umi @@ -48,14 +52,503 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(workflow.out.reads).match("reads") }, - { assert snapshot(workflow.out.umi_log).match("umi_log") }, - { assert snapshot(workflow.out.trim_json).match("trim_json") }, - { assert snapshot(workflow.out.trim_reads_fail).match("trim_reads_fail") }, - { assert snapshot(workflow.out.trim_reads_merged).match("trim_reads_merged") }, - { assert snapshot(workflow.out.adapter_seq).match("adapter_seq") }, - { assert snapshot(workflow.out.trim_read_count).match("trim_read_count") }, - { assert snapshot(workflow.out.versions).match("versions") }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("skip_fastqc") { + + when { + workflow { + """ + skip_fastqc = true + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end: false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert !workflow.out.fastqc_raw_html }, + { assert !workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert !workflow.out.fastqc_trim_html }, + { assert !workflow.out.fastqc_trim_zip } + ) + } + } + + test("with_umi") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = true + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + + test("skip_umi_extract") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = true + skip_umi_extract = true + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("umi_discard_read = 2") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = true + skip_umi_extract = true + umi_discard_read = 2 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("skip_trimming") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = true + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads.get(0).get(0), // Reads meta map + // Because the input file is passed to the output file, we have to do check the filename only + file(workflow.out.reads.get(0).get(1).get(0)).name, + file(workflow.out.reads.get(0).get(1).get(1)).name, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert !workflow.out.trim_html }, + { assert !workflow.out.trim_log }, + { assert !workflow.out.fastqc_trim_html }, + { assert !workflow.out.fastqc_trim_zip } + ) + } + } + + test("save_trimmed_fail") { + + config './nextflow.save_trimmed.config' + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = true + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("save_merged") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = true + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("min_trimmed_reads = 26") { + // Subworkflow should stop after FASTP which trims down to 25 reads + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = true + min_trimmed_reads = 26 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, { assert workflow.out.fastqc_raw_html }, { assert workflow.out.fastqc_raw_zip }, diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap index 89ba8da1..3e11d9ec 100644 --- a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap @@ -1,30 +1,186 @@ { - "trim_reads_merged": { + "skip_fastqc": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,67b2bbae47f073e05a97a9c2edce23c7", + "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" + ] + ], [ + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.041635573" + "timestamp": "2024-03-18T16:53:49.315194" }, - "trim_reads_fail": { + "save_trimmed_fail": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,6ff32a64c5188b9a9192be1398c262c7", + "test_2.fastp.fastq.gz:md5,db0cb7c9977e94ac2b4b446ebd017a8a" + ] + ] + ], [ + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,4c3268ddb50ea5b33125984776aa3519" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test.paired.fail.fastq.gz:md5,409b687c734cedd7a1fec14d316e1366", + "test_1.fail.fastq.gz:md5,4f273cf3159c13f79e8ffae12f5661f6", + "test_2.fail.fastq.gz:md5,f97b9edefb5649aab661fbc9e71fc995" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 162 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.033284693" + "timestamp": "2024-03-18T16:51:45.34934" }, - "versions": { + "skip_umi_extract": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,67b2bbae47f073e05a97a9c2edce23c7", + "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" + ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], [ "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", @@ -35,10 +191,25 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.121510557" + "timestamp": "2024-03-18T12:07:40.34249" }, - "trim_json": { + "umi_discard_read = 2": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,67b2bbae47f073e05a97a9c2edce23c7", + "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" + ] + ] + ], + [ + + ], [ [ { @@ -47,16 +218,81 @@ }, "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.024410724" + "timestamp": "2024-03-18T12:08:24.141938" }, - "adapter_seq": { + "save_merged": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,54b726a55e992a869fd3fa778afe1672", + "test_2.fastp.fastq.gz:md5,29d3b33b869f7b63417b8ff07bb128ba" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,b712fd68ed0322f4bec49ff2a5237fcc" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.merged.fastq.gz:md5,c873bb1ab3fa859dcc47306465e749d5" + ] + ], [ [ { @@ -65,15 +301,65 @@ }, "unspecified" ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 75 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.08674429" + "timestamp": "2024-03-18T12:10:18.546963" }, - "reads": { + "skip_trimming": { + "content": [ + { + "id": "test", + "single_end": false + }, + "test_1.fastq.gz", + "test_2.fastq.gz", + [ + + ], + [ + + ], + [ + + ], + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T15:49:26.574759" + }, + "sarscov2 paired-end [fastq]": { "content": [ [ [ @@ -86,42 +372,178 @@ "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" ] ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" + ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:49.994419936" + "timestamp": "2024-03-18T16:53:39.139038" }, - "umi_log": { + "min_trimmed_reads = 26": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,54b726a55e992a869fd3fa778afe1672", + "test_2.fastp.fastq.gz:md5,29d3b33b869f7b63417b8ff07bb128ba" + ] + ] + ], [ + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,b712fd68ed0322f4bec49ff2a5237fcc" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.merged.fastq.gz:md5,c873bb1ab3fa859dcc47306465e749d5" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 75 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.017720214" + "timestamp": "2024-03-18T11:52:23.849945" }, - "trim_read_count": { + "with_umi": { "content": [ [ [ { "id": "test", - "single_end": false + "single_end": true }, - 198 + "test.fastp.fastq.gz:md5,ba8c6c3a7ce718d9a2c5857e2edf53bc" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.fastp.json:md5,d39c5c6d9a2e35fb60d26ced46569af6" ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + 99 + ] + ], + [ + "versions.yml:md5,01f264f78de3c6d893c449cc6d3cd721", + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.102326089" + "timestamp": "2024-03-18T17:31:09.193212" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config new file mode 100644 index 00000000..12f7b257 --- /dev/null +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config @@ -0,0 +1,11 @@ +process { + + withName: UMITOOLS_EXTRACT { + ext.args = '--bc-pattern="NNNN" --bc-pattern2="NNNN"' + } + + withName: UMICOLLAPSE { + ext.prefix = { "${meta.id}.dedup" } + } + +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config new file mode 100644 index 00000000..2430e9d5 --- /dev/null +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config @@ -0,0 +1,6 @@ +process { + // Make filtering more aggressive to make more reads fail + withName: FASTP { + ext.args = "-e 30" + } +} \ No newline at end of file