From 89ace606d09bfe97e195281095c4427fe9ab12fe Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Feb 2024 13:40:08 +0000 Subject: [PATCH 01/32] Bump verison in dev --- CHANGELOG.md | 4 ++++ assets/multiqc_config.yml | 4 ++-- nextflow.config | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48e2675b..ccf6f391 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v2.3.1 - XXXX-XX-XX- Gray Zinc Dalmation Patch + +- _tbd_ + ## v2.3.0 - 2024-02-23 - Gray Zinc Dalmatian - [[#307]](https://github.com/nf-core/smrnaseq/pull/307) - Clean up config file and improve output folder structure diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index cb6b62c8..2b3c2bd9 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/smrnaseq + This report has been generated by the nf-core/smrnaseq analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-smrnaseq-methods-description": diff --git a/nextflow.config b/nextflow.config index 272b7372..9ff0dd24 100644 --- a/nextflow.config +++ b/nextflow.config @@ -305,7 +305,7 @@ manifest { description = """Small RNA-Seq Best Practice Analysis Pipeline.""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '2.3.0' + version = '2.3.1dev' doi = '10.5281/zenodo.3456879' } From c3d474d6141447a6d4f80872c31fbdfe1a27f0a3 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Feb 2024 15:13:38 +0100 Subject: [PATCH 02/32] Update nextflow.config Co-authored-by: Maxime U Garcia --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 9ff0dd24..6fd4c05d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -305,7 +305,7 @@ manifest { description = """Small RNA-Seq Best Practice Analysis Pipeline.""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '2.3.1dev' + version = '2.3.1dev' doi = '10.5281/zenodo.3456879' } From 336bd5b293def3e499b95408a0f2b3d8dc5dc7e4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 29 Feb 2024 08:32:12 +0000 Subject: [PATCH 03/32] Upgrade MQC 1.21 --- CHANGELOG.md | 6 ++ modules.json | 4 +- modules/nf-core/cat/cat/tests/main.nf.test | 29 +++--- .../nf-core/cat/cat/tests/main.nf.test.snap | 92 +++++++------------ modules/nf-core/multiqc/environment.yml | 2 +- modules/nf-core/multiqc/main.nf | 4 +- .../nf-core/multiqc/tests/main.nf.test.snap | 12 +-- 7 files changed, 66 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf6f391..1ede37a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - _tbd_ +### Software dependencies + +| Dependency | Old version | New version | +| ---------- | ----------- | ----------- | +| `multiqc` | 1.20 | 1.21 | + ## v2.3.0 - 2024-02-23 - Gray Zinc Dalmatian - [[#307]](https://github.com/nf-core/smrnaseq/pull/307) - Clean up config file and improve output folder structure diff --git a/modules.json b/modules.json index ad13815e..2f860e63 100644 --- a/modules.json +++ b/modules.json @@ -7,7 +7,7 @@ "nf-core": { "cat/cat": { "branch": "master", - "git_sha": "81f27e75847087865299cc46605deb3b09b4e0a2", + "git_sha": "9437e6053dccf4aafa022bfd6e7e9de67e625af8", "installed_by": ["modules"] }, "cat/fastq": { @@ -27,7 +27,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "ccacf6f5de6df3bc6d73b665c1fd2933d8bbc290", + "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", "installed_by": ["modules"] }, "samtools/flagstat": { diff --git a/modules/nf-core/cat/cat/tests/main.nf.test b/modules/nf-core/cat/cat/tests/main.nf.test index aaae04f9..fcee2d19 100644 --- a/modules/nf-core/cat/cat/tests/main.nf.test +++ b/modules/nf-core/cat/cat/tests/main.nf.test @@ -19,8 +19,8 @@ nextflow_process { [ [ id:'genome', single_end:true ], [ - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['genome']['genome_sizes'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.sizes', checkIfExists: true) ] ] """ @@ -45,8 +45,8 @@ nextflow_process { [ [ id:'test', single_end:true ], [ - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['genome']['genome_sizes'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.sizes', checkIfExists: true) ] ] """ @@ -72,8 +72,8 @@ nextflow_process { [ [ id:'test', single_end:true ], [ - file(params.test_data['sarscov2']['genome']['genome_gff3_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['genome']['contigs_genome_maf_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.gff3.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/alignment/last/contigs.genome.maf.gz', checkIfExists: true) ] ] """ @@ -83,7 +83,8 @@ nextflow_process { def lines = path(process.out.file_out.get(0).get(1)).linesGzip assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(lines[0..5]).match("test_cat_zipped_zipped_lines") }, + { assert snapshot(lines.size()).match("test_cat_zipped_zipped_size")} ) } } @@ -101,8 +102,8 @@ nextflow_process { [ [ id:'test', single_end:true ], [ - file(params.test_data['sarscov2']['genome']['genome_gff3_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['genome']['contigs_genome_maf_gz'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.gff3.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/alignment/last/contigs.genome.maf.gz', checkIfExists: true) ] ] """ @@ -130,8 +131,8 @@ nextflow_process { [ [ id:'test', single_end:true ], [ - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), - file(params.test_data['sarscov2']['genome']['genome_sizes'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.sizes', checkIfExists: true) ] ] """ @@ -141,7 +142,8 @@ nextflow_process { def lines = path(process.out.file_out.get(0).get(1)).linesGzip assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(lines[0..5]).match("test_cat_unzipped_zipped_lines") }, + { assert snapshot(lines.size()).match("test_cat_unzipped_zipped_size")} ) } } @@ -158,7 +160,7 @@ nextflow_process { [ [ id:'test', single_end:true ], [ - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ] ] """ @@ -174,4 +176,3 @@ nextflow_process { } } } - diff --git a/modules/nf-core/cat/cat/tests/main.nf.test.snap b/modules/nf-core/cat/cat/tests/main.nf.test.snap index 0c9bfe8d..423571ba 100644 --- a/modules/nf-core/cat/cat/tests/main.nf.test.snap +++ b/modules/nf-core/cat/cat/tests/main.nf.test.snap @@ -1,4 +1,10 @@ { + "test_cat_unzipped_zipped_size": { + "content": [ + 375 + ], + "timestamp": "2023-10-16T14:33:08.049445686" + }, "test_cat_unzipped_unzipped": { "content": [ { @@ -61,36 +67,31 @@ ], "timestamp": "2023-10-16T14:32:49.642741302" }, - "test_cat_zipped_zipped": { + "test_cat_zipped_zipped_lines": { "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": true - }, - "test.gff3.gz:md5,c439d3b60e7bc03e8802a451a0d9a5d9" - ] - ], - "1": [ - "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" - ], - "file_out": [ - [ - { - "id": "test", - "single_end": true - }, - "test.gff3.gz:md5,c439d3b60e7bc03e8802a451a0d9a5d9" - ] - ], - "versions": [ - "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" - ] - } + [ + "MT192765.1\tGenbank\ttranscript\t259\t29667\t.\t+\t.\tID=unknown_transcript_1;geneID=orf1ab;gene_name=orf1ab", + "MT192765.1\tGenbank\tgene\t259\t21548\t.\t+\t.\tParent=unknown_transcript_1", + "MT192765.1\tGenbank\tCDS\t259\t13461\t.\t+\t0\tParent=unknown_transcript_1;exception=\"ribosomal slippage\";gbkey=CDS;gene=orf1ab;note=\"pp1ab;translated=by -1 ribosomal frameshift\";product=\"orf1ab polyprotein\";protein_id=QIK50426.1", + "MT192765.1\tGenbank\tCDS\t13461\t21548\t.\t+\t0\tParent=unknown_transcript_1;exception=\"ribosomal slippage\";gbkey=CDS;gene=orf1ab;note=\"pp1ab;translated=by -1 ribosomal frameshift\";product=\"orf1ab polyprotein\";protein_id=QIK50426.1", + "MT192765.1\tGenbank\tCDS\t21556\t25377\t.\t+\t0\tParent=unknown_transcript_1;gbkey=CDS;gene=S;note=\"structural protein\";product=\"surface glycoprotein\";protein_id=QIK50427.1", + "MT192765.1\tGenbank\tgene\t21556\t25377\t.\t+\t.\tParent=unknown_transcript_1" + ] + ], + "timestamp": "2023-10-16T14:32:33.629048645" + }, + "test_cat_unzipped_zipped_lines": { + "content": [ + [ + ">MT192765.1 Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/USA/PC00101P/2020, complete genome", + "GTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGT", + "GTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAG", + "TAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTTGTCCGG", + "GTGTGACCGAAAGGTAAGATGGAGAGCCTTGTCCCTGGTTTCAACGAGAAAACACACGTCCAACTCAGTTTGCCTGTTTT", + "ACAGGTTCGCGACGTGCTCGTACGTGGCTTTGGAGACTCCGTGGAGGAGGTCTTATCAGAGGCACGTCAACATCTTAAAG" + ] ], - "timestamp": "2024-01-12T14:02:02.999254641" + "timestamp": "2023-10-16T14:33:08.038830506" }, "test_cat_one_file_unzipped_zipped_lines": { "content": [ @@ -105,41 +106,16 @@ ], "timestamp": "2023-10-16T14:33:21.39642399" }, - "test_cat_unzipped_zipped": { + "test_cat_zipped_zipped_size": { "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": true - }, - "cat.txt.gz:md5,f44b33a0e441ad58b2d3700270e2dbe2" - ] - ], - "1": [ - "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" - ], - "file_out": [ - [ - { - "id": "test", - "single_end": true - }, - "cat.txt.gz:md5,f44b33a0e441ad58b2d3700270e2dbe2" - ] - ], - "versions": [ - "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" - ] - } + 78 ], - "timestamp": "2024-01-12T14:08:26.948048418" + "timestamp": "2023-10-16T14:32:33.641869244" }, "test_cat_one_file_unzipped_zipped_size": { "content": [ 374 ], - "timestamp": "2024-01-12T14:10:22.445700266" + "timestamp": "2023-10-16T14:33:21.4094373" } -} +} \ No newline at end of file diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index 2212096a..ca39fb67 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.20 + - bioconda::multiqc=1.21 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 354f4430..47ac352f 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.20--pyhdfd78af_0' : - 'biocontainers/multiqc:1.20--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : + 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index c204b488..bfebd802 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-14T09:28:51.744211298" + "timestamp": "2024-02-29T08:48:55.657331" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-14T09:29:28.847433492" + "timestamp": "2024-02-29T08:49:49.071937" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-14T09:29:13.223621555" + "timestamp": "2024-02-29T08:49:25.457567" } } \ No newline at end of file From 1e13a30be5939392d8b634cf4a1052025b4a85a2 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 29 Feb 2024 16:12:56 +0000 Subject: [PATCH 04/32] Template update for nf-core/tools version 2.13.1 --- .devcontainer/devcontainer.json | 10 +---- .github/CONTRIBUTING.md | 14 ++++--- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/awsfulltest.yml | 4 +- .github/workflows/awstest.yml | 4 +- .github/workflows/ci.yml | 2 +- .github/workflows/download_pipeline.yml | 2 +- .github/workflows/linting.yml | 2 +- .github/workflows/release-announcements.yml | 2 +- .gitpod.yml | 6 +-- CHANGELOG.md | 2 +- README.md | 3 +- assets/multiqc_config.yml | 6 +-- modules.json | 8 ++-- modules/nf-core/multiqc/environment.yml | 2 +- modules/nf-core/multiqc/main.nf | 4 +- .../nf-core/multiqc/tests/main.nf.test.snap | 12 +++--- nextflow.config | 2 +- .../utils_nfcore_smrnaseq_pipeline/main.nf | 10 +++-- .../tests/main.function.nf.test | 2 +- .../tests/main.function.nf.test.snap | 12 +++++- .../tests/main.workflow.nf.test | 20 ++------- .../tests/nextflow.config | 2 +- .../tests/main.function.nf.test.snap | 42 +++++++++++++++---- .../tests/main.workflow.nf.test.snap | 6 ++- .../tests/main.nf.test | 2 +- 26 files changed, 103 insertions(+), 80 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe3..b290e090 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,15 +10,7 @@ "vscode": { // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.defaultInterpreterPath": "/opt/conda/bin/python" }, // Add the IDs of extensions you want installed when the container is created. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index bd134c88..0779fb9b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -9,9 +9,8 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -:::info -If you need help using or modifying nf-core/smrnaseq then the best place to ask is on the nf-core Slack [#smrnaseq](https://nfcore.slack.com/channels/smrnaseq) channel ([join our Slack here](https://nf-co.re/join/slack)). -::: +> [!NOTE] +> If you need help using or modifying nf-core/smrnaseq then the best place to ask is on the nf-core Slack [#smrnaseq](https://nfcore.slack.com/channels/smrnaseq) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow @@ -27,8 +26,11 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests -You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to -receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. +You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: + +```bash +nf-test test --profile debug,test,docker --verbose +``` When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. @@ -90,7 +92,7 @@ Once there, use `nf-core schema build` to add to `nextflow_schema.json`. Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. -The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block. +The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block. ### Naming schemes diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3cdbf2b7..ef59ff45 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/smrn - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/smrnaseq/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/smrnaseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 3c8cf6d6..b849392a 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@922e5c8d5ac4e918107ec311d2ebbd65e5982b3d # v2 + uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters @@ -31,7 +31,7 @@ jobs: } profiles: test_full - - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + - uses: actions/upload-artifact@v4 with: name: Tower debug log file path: | diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index 3ed287c5..5386cbc0 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@922e5c8d5ac4e918107ec311d2ebbd65e5982b3d # v2 + uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} @@ -25,7 +25,7 @@ jobs: } profiles: test - - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + - uses: actions/upload-artifact@v4 with: name: Tower debug log file path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bd73b88..d34af95d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 + uses: nf-core/setup-nextflow@v1 with: version: "${{ matrix.NXF_VER }}" diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index f823210d..08622fd5 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 748b4311..073e1876 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/.github/workflows/release-announcements.yml b/.github/workflows/release-announcements.yml index c3674af2..d468aeaa 100644 --- a/.github/workflows/release-announcements.yml +++ b/.github/workflows/release-announcements.yml @@ -12,7 +12,7 @@ jobs: - name: get topics and convert to hashtags id: get_topics run: | - curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' > $GITHUB_OUTPUT + curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT - uses: rzr/fediverse-action@master with: diff --git a/.gitpod.yml b/.gitpod.yml index 363d5b1d..105a1821 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -10,13 +10,11 @@ tasks: vscode: extensions: # based on nf-core.nf-core-extensionpack - - codezombiech.gitignore # Language support for .gitignore files - # - cssho.vscode-svgviewer # SVG viewer - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code + - charliermarsh.ruff # Code linter Ruff diff --git a/CHANGELOG.md b/CHANGELOG.md index bca9b0c2..9ad620bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.3.0 - [date] +## v2.3.1dev - [date] Initial release of nf-core/smrnaseq, created with the [nf-core](https://nf-co.re/) template. diff --git a/README.md b/README.md index e38215dd..ce785dbb 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,13 @@ [![GitHub Actions CI Status](https://github.com/nf-core/smrnaseq/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/ci.yml) [![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/smrnaseq) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/smrnaseq) [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23smrnaseq-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/smrnaseq)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index c2cfbb98..bd6b5ac7 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,9 +1,7 @@ report_comment: > - - This report has been generated by the nf-core/smrnaseq + This report has been generated by the nf-core/smrnaseq analysis pipeline. For information about how to interpret these results, please see the - documentation. - + documentation. report_section_order: "nf-core-smrnaseq-methods-description": order: -1000 diff --git a/modules.json b/modules.json index 812aef49..09b1428e 100644 --- a/modules.json +++ b/modules.json @@ -12,7 +12,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "ccacf6f5de6df3bc6d73b665c1fd2933d8bbc290", + "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", "installed_by": ["modules"] } } @@ -21,17 +21,17 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "262b17ed2aad591039f914951659177e6c39a8d8", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { "branch": "master", - "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] } } diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index 2212096a..ca39fb67 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.20 + - bioconda::multiqc=1.21 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 354f4430..47ac352f 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.20--pyhdfd78af_0' : - 'biocontainers/multiqc:1.20--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : + 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index c204b488..bfebd802 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-14T09:28:51.744211298" + "timestamp": "2024-02-29T08:48:55.657331" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-14T09:29:28.847433492" + "timestamp": "2024-02-29T08:49:49.071937" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-14T09:29:13.223621555" + "timestamp": "2024-02-29T08:49:25.457567" } } \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 8aa09a78..1ef2a099 100644 --- a/nextflow.config +++ b/nextflow.config @@ -231,7 +231,7 @@ manifest { description = """Small RNA-Seq Best Practice Analysis Pipeline.""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '2.3.0' + version = '2.3.1dev' doi = '' } diff --git a/subworkflows/local/utils_nfcore_smrnaseq_pipeline/main.nf b/subworkflows/local/utils_nfcore_smrnaseq_pipeline/main.nf index dc90ad90..108bad19 100644 --- a/subworkflows/local/utils_nfcore_smrnaseq_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_smrnaseq_pipeline/main.nf @@ -1,5 +1,5 @@ // -// Subworkflow with functionality specific to the nf-core/pipeline pipeline +// Subworkflow with functionality specific to the nf-core/smrnaseq pipeline // /* @@ -152,7 +152,9 @@ workflow PIPELINE_COMPLETION { // def validateInputParameters() { genomeExistsError() -}// +} + +// // Validate channels from input samplesheet // def validateInputSamplesheet(input) { @@ -190,7 +192,9 @@ def genomeExistsError() { "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" error(error_string) } -}// +} + +// // Generate methods description for MultiQC // def toolCitationText() { diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test index 8ed4310c..68718e4f 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -51,4 +51,4 @@ nextflow_function { ) } } -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap index db2030f8..e3f0baf4 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -3,10 +3,18 @@ "content": [ "v9.9.9" ], - "timestamp": "2024-01-19T11:32:36.031083" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:05.308243" }, "Test Function checkCondaChannels": { "content": null, - "timestamp": "2024-01-19T11:32:50.456" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:12.425833" } } \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test index f7c54bc6..ca964ce8 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -11,9 +11,6 @@ nextflow_workflow { test("Should run no inputs") { when { - params { - outdir = "tests/results" - } workflow { """ print_version = false @@ -39,9 +36,6 @@ nextflow_workflow { test("Should print version") { when { - params { - outdir = "tests/results" - } workflow { """ print_version = true @@ -68,19 +62,16 @@ nextflow_workflow { test("Should dump params") { when { - params { - outdir = "$outputDir" - } workflow { """ print_version = false dump_parameters = true - outdir = params.outdir + outdir = 'results' check_conda_channels = false input[0] = false input[1] = true - input[2] = params.outdir + input[2] = outdir input[3] = false """ } @@ -96,19 +87,16 @@ nextflow_workflow { test("Should not create params JSON if no output directory") { when { - params { - outdir = "$outputDir" - } workflow { """ print_version = false dump_parameters = true - outdir = params.outdir + outdir = null check_conda_channels = false input[0] = false input[1] = true - input[2] = null + input[2] = outdir input[3] = false """ } diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index 53574ffe..d0a926bf 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -6,4 +6,4 @@ manifest { nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap index 10f948e6..1037232c 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -1,25 +1,41 @@ { "Test Function checkProfileProvided": { "content": null, - "timestamp": "2024-02-09T15:43:55.145717" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" }, "Test Function checkConfigProvided": { "content": [ true ], - "timestamp": "2024-01-19T11:34:13.548431224" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" }, "Test Function nfCoreLogo": { "content": [ "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" ], - "timestamp": "2024-01-19T11:34:38.840454873" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:10.562934" }, "Test Function workflowCitation": { "content": [ "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" ], - "timestamp": "2024-01-19T11:34:22.24352016" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:07.019761" }, "Test Function without logColours": { "content": [ @@ -73,13 +89,21 @@ "biwhite": "" } ], - "timestamp": "2024-01-19T11:35:04.418416984" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" }, "Test Function dashedLine": { "content": [ "-\u001b[2m----------------------------------------------------\u001b[0m-" ], - "timestamp": "2024-01-19T11:34:55.420000755" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:14.366181" }, "Test Function with logColours": { "content": [ @@ -133,6 +157,10 @@ "biwhite": "\u001b[1;97m" } ], - "timestamp": "2024-01-19T11:35:13.436366565" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" } } \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap index d07ce54c..859d1030 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -10,6 +10,10 @@ ] } ], - "timestamp": "2024-01-19T11:35:22.538940073" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" } } \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test index 517ee54e..5784a33f 100644 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -197,4 +197,4 @@ nextflow_workflow { ) } } -} \ No newline at end of file +} From 0ead8cd041bf2b903ce30f9b25fa194c5612ec04 Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Tue, 19 Mar 2024 17:16:10 +0000 Subject: [PATCH 05/32] change type of reads input --- modules/local/mirtrace.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index 500de058..470ec95c 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -7,7 +7,7 @@ process MIRTRACE_RUN { 'biocontainers/mirtrace:1.0.1--hdfd78af_1' }" input: - tuple val(adapter), val(ids), path(reads) + tuple val(adapter), val(ids), val(reads) output: path "mirtrace/*" , emit: mirtrace From 2ad0c27139509b11657b98d926e971c7e302a231 Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Tue, 19 Mar 2024 17:29:45 +0000 Subject: [PATCH 06/32] reinstall bam_sort_stats_samtools --- modules.json | 2 +- subworkflows/nf-core/bam_sort_stats_samtools/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules.json b/modules.json index 22166d17..8f374d39 100644 --- a/modules.json +++ b/modules.json @@ -76,7 +76,7 @@ "nf-core": { "bam_sort_stats_samtools": { "branch": "master", - "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", + "git_sha": "4352dbdb09ec40db71e9b172b97a01dcf5622c26", "installed_by": ["subworkflows"] }, "bam_stats_samtools": { diff --git a/subworkflows/nf-core/bam_sort_stats_samtools/main.nf b/subworkflows/nf-core/bam_sort_stats_samtools/main.nf index fc1c652b..b716375b 100644 --- a/subworkflows/nf-core/bam_sort_stats_samtools/main.nf +++ b/subworkflows/nf-core/bam_sort_stats_samtools/main.nf @@ -15,7 +15,7 @@ workflow BAM_SORT_STATS_SAMTOOLS { ch_versions = Channel.empty() - SAMTOOLS_SORT ( ch_bam ) + SAMTOOLS_SORT ( ch_bam, ch_fasta ) ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions.first()) SAMTOOLS_INDEX ( SAMTOOLS_SORT.out.bam ) From 4ec45d01ec9b33d750a22cc1ad55b700130fc003 Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Tue, 19 Mar 2024 17:34:25 +0000 Subject: [PATCH 07/32] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ede37a6..6a3496b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## v2.3.1 - XXXX-XX-XX- Gray Zinc Dalmation Patch -- _tbd_ +- [[#328]](https://github.com/nf-core/smrnaseq/pull/328) - Fix [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module ### Software dependencies From 684a84b1c3ced7049789772983aee8827003f93d Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Tue, 19 Mar 2024 17:52:23 +0000 Subject: [PATCH 08/32] update samtools sort module --- modules.json | 2 +- modules/nf-core/samtools/sort/main.nf | 28 ++++-- modules/nf-core/samtools/sort/meta.yml | 32 ++++-- .../nf-core/samtools/sort/tests/main.nf.test | 38 ++++++- .../samtools/sort/tests/main.nf.test.snap | 99 ++++++++++++++++++- .../samtools/sort/tests/nextflow.config | 3 +- 6 files changed, 179 insertions(+), 23 deletions(-) diff --git a/modules.json b/modules.json index 8f374d39..1d31a680 100644 --- a/modules.json +++ b/modules.json @@ -47,7 +47,7 @@ }, "samtools/sort": { "branch": "master", - "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", + "git_sha": "4352dbdb09ec40db71e9b172b97a01dcf5622c26", "installed_by": ["bam_sort_stats_samtools", "modules"] }, "samtools/stats": { diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf index cdd8305d..fc374f98 100644 --- a/modules/nf-core/samtools/sort/main.nf +++ b/modules/nf-core/samtools/sort/main.nf @@ -8,11 +8,14 @@ process SAMTOOLS_SORT { 'biocontainers/samtools:1.19.2--h50ea8bc_0' }" input: - tuple val(meta), path(bam) + tuple val(meta) , path(bam) + tuple val(meta2), path(fasta) output: - tuple val(meta), path("*.bam"), emit: bam - tuple val(meta), path("*.csi"), emit: csi, optional: true + tuple val(meta), path("*.bam"), emit: bam, optional: true + tuple val(meta), path("*.cram"), emit: cram, optional: true + tuple val(meta), path("*.crai"), emit: crai, optional: true + tuple val(meta), path("*.csi"), emit: csi, optional: true path "versions.yml" , emit: versions when: @@ -21,14 +24,24 @@ process SAMTOOLS_SORT { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def extension = args.contains("--output-fmt sam") ? "sam" : + args.contains("--output-fmt cram") ? "cram" : + "bam" + def reference = fasta ? "--reference ${fasta}" : "" if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" + """ + samtools cat \\ + --threads $task.cpus \\ + ${bam} \\ + | \\ samtools sort \\ $args \\ - -@ $task.cpus \\ - -o ${prefix}.bam \\ - -T $prefix \\ - $bam + -T ${prefix} \\ + --threads $task.cpus \\ + ${reference} \\ + -o ${prefix}.${extension} \\ + - cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -40,6 +53,7 @@ process SAMTOOLS_SORT { def prefix = task.ext.prefix ?: "${meta.id}" """ touch ${prefix}.bam + touch ${prefix}.bam.csi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/samtools/sort/meta.yml b/modules/nf-core/samtools/sort/meta.yml index 2200de72..341a7d0e 100644 --- a/modules/nf-core/samtools/sort/meta.yml +++ b/modules/nf-core/samtools/sort/meta.yml @@ -23,8 +23,18 @@ input: e.g. [ id:'test', single_end:false ] - bam: type: file - description: BAM/CRAM/SAM file + description: BAM/CRAM/SAM file(s) pattern: "*.{bam,cram,sam}" + - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] + - fasta: + type: file + description: Reference genome FASTA file + pattern: "*.{fa,fasta,fna}" + optional: true output: - meta: type: map @@ -33,19 +43,29 @@ output: e.g. [ id:'test', single_end:false ] - bam: type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - - versions: + description: Sorted BAM file + pattern: "*.{bam}" + - cram: type: file - description: File containing software versions - pattern: "versions.yml" + description: Sorted CRAM file + pattern: "*.{cram}" + - crai: + type: file + description: CRAM index file (optional) + pattern: "*.crai" - csi: type: file description: BAM index file (optional) pattern: "*.csi" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@ewels" + - "@matthdsm" maintainers: - "@drpatelh" - "@ewels" + - "@matthdsm" diff --git a/modules/nf-core/samtools/sort/tests/main.nf.test b/modules/nf-core/samtools/sort/tests/main.nf.test index 31e24b88..8360e2b1 100644 --- a/modules/nf-core/samtools/sort/tests/main.nf.test +++ b/modules/nf-core/samtools/sort/tests/main.nf.test @@ -13,15 +13,43 @@ nextflow_process { config "./nextflow.config" when { - params { - outdir = "$outputDir" + process { + """ + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) + ]) + input[1] = Channel.of([ + [ id:'fasta' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + """ } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("cram") { + + config "./nextflow.config" + + when { process { """ input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) ]) + input[1] = Channel.of([ + [ id:'fasta' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } @@ -37,7 +65,7 @@ nextflow_process { test("bam_stub") { config "./nextflow.config" - options "-stub-run" + options "-stub" when { params { @@ -49,6 +77,10 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true) ]) + input[1] = Channel.of([ + [ id:'fasta' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } diff --git a/modules/nf-core/samtools/sort/tests/main.nf.test.snap b/modules/nf-core/samtools/sort/tests/main.nf.test.snap index a7cf0210..38477656 100644 --- a/modules/nf-core/samtools/sort/tests/main.nf.test.snap +++ b/modules/nf-core/samtools/sort/tests/main.nf.test.snap @@ -1,4 +1,69 @@ { + "cram": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" + ] + ], + "1": [ + + ], + "2": [ + + ], + "3": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] + ], + "4": [ + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" + ], + "bam": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" + ] + ], + "crai": [ + + ], + "cram": [ + + ], + "csi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] + ], + "versions": [ + "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-04T15:08:00.830294" + }, "bam_stub_bam": { "content": [ "test.sorted.bam" @@ -30,13 +95,25 @@ "id": "test", "single_end": false }, - "test.sorted.bam:md5,c6ea1346ec4aae007eb40b708935088c" + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" ] ], "1": [ ], "2": [ + + ], + "3": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] + ], + "4": [ "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" ], "bam": [ @@ -45,11 +122,23 @@ "id": "test", "single_end": false }, - "test.sorted.bam:md5,c6ea1346ec4aae007eb40b708935088c" + "test.sorted.bam:md5,bc0b7c25da26384a006ed84cc9e4da23" ] ], - "csi": [ + "crai": [ + + ], + "cram": [ + ], + "csi": [ + [ + { + "id": "test", + "single_end": false + }, + "test.sorted.bam.csi:md5,8d4e836c2fed6c0bf874d5e8cdba5831" + ] ], "versions": [ "versions.yml:md5,e6d43fefc9a8bff91c2ce6e3a1716eca" @@ -58,8 +147,8 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-13T16:14:52.736359271" + "timestamp": "2024-03-04T15:07:48.773803" } } \ No newline at end of file diff --git a/modules/nf-core/samtools/sort/tests/nextflow.config b/modules/nf-core/samtools/sort/tests/nextflow.config index d0f35086..f642771f 100644 --- a/modules/nf-core/samtools/sort/tests/nextflow.config +++ b/modules/nf-core/samtools/sort/tests/nextflow.config @@ -1,7 +1,8 @@ process { withName: SAMTOOLS_SORT { - ext.prefix = { "${meta.id}.sorted" } + ext.prefix = { "${meta.id}.sorted" } + ext.args = "--write-index" } } From fc1c527a991cddc95af401327eef5360c11e6f41 Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Tue, 2 Apr 2024 08:42:30 +0000 Subject: [PATCH 09/32] fix input channel cardinality --- modules/local/mirdeep2_run.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/mirdeep2_run.nf b/modules/local/mirdeep2_run.nf index 442f26f3..ba37a4ac 100644 --- a/modules/local/mirdeep2_run.nf +++ b/modules/local/mirdeep2_run.nf @@ -12,8 +12,8 @@ process MIRDEEP2_RUN { input: path(fasta) tuple path(reads), path(arf) - tuple val(meta2), path(hairpin) - tuple val(meta2), path(mature) + path(hairpin) + path(mature) output: path 'result*.{bed,csv,html}', emit: result From 752aeea0598626d7fc8641372631c2321fb48e3c Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Tue, 2 Apr 2024 11:02:03 +0000 Subject: [PATCH 10/32] fix conda bowtie version --- modules/local/bowtie_map_mirna.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bowtie_map_mirna.nf b/modules/local/bowtie_map_mirna.nf index 148b47f5..d6b0ea8f 100644 --- a/modules/local/bowtie_map_mirna.nf +++ b/modules/local/bowtie_map_mirna.nf @@ -2,7 +2,7 @@ process BOWTIE_MAP_SEQ { tag "$meta.id" label 'process_medium' - conda 'bowtie=1.3.0-2 bioconda::samtools=1.13' + conda 'bowtie=1.3.0 bioconda::samtools=1.13' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:40128b496751b037e2bd85f6789e83d4ff8a4837-0' : 'biocontainers/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:40128b496751b037e2bd85f6789e83d4ff8a4837-0' }" From f3df2e3757e34d31a8eb422a11c6f92c47b4bace Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Wed, 3 Apr 2024 07:57:54 +0000 Subject: [PATCH 11/32] add changes to CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3496b1..6e814491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## v2.3.1 - XXXX-XX-XX- Gray Zinc Dalmation Patch - [[#328]](https://github.com/nf-core/smrnaseq/pull/328) - Fix [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module +- [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [input channel cardinality](https://github.com/nf-core/smrnaseq/issues/331) in `MIRDEEP2_RUN` module +- [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [bowtie conda version](https://github.com/nf-core/smrnaseq/issues/333) in `BOWTIE_MAP_SEQ` module ### Software dependencies From cae7daed111af2c0690230f2d1f23dd3b935432f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 3 Apr 2024 08:16:17 +0000 Subject: [PATCH 12/32] Hopeful fix for mirtrace casting & staging bug --- CHANGELOG.md | 1 + modules/local/mirtrace.nf | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3496b1..a0b50098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## v2.3.1 - XXXX-XX-XX- Gray Zinc Dalmation Patch - [[#328]](https://github.com/nf-core/smrnaseq/pull/328) - Fix [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module +- [[#331]](https://github.com/nf-core/smrnaseq/pull/331) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module ### Software dependencies diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index 470ec95c..3641a97c 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -7,7 +7,7 @@ process MIRTRACE_RUN { 'biocontainers/mirtrace:1.0.1--hdfd78af_1' }" input: - tuple val(adapter), val(ids), val(reads) + tuple val(adapter), val(ids), path(reads) output: path "mirtrace/*" , emit: mirtrace @@ -25,9 +25,13 @@ process MIRTRACE_RUN { tmem = task.memory.toBytes() java_mem = "-Xms${tmem} -Xmx${tmem}" } + + //Staging the files as path() but then getting the filenames for the config file that mirtrace needs + //Directly using val(reads) as in previous versions is not reliable as staging between work directories is not 100% reliable if not explicitly defined via nextflow itself def config_lines = [ids,reads] .transpose() - .collect({ id, path -> "echo '${path},${id}' >> mirtrace_config" }) + .collect({ id, path -> path.getFileName().toString()}) + """ export mirtracejar=\$(dirname \$(which mirtrace)) From 4e80dedf11d0f82b48ef01a7c9cd92c789826511 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 3 Apr 2024 08:29:57 +0000 Subject: [PATCH 13/32] Not working, but pushing for later --- modules/local/mirtrace.nf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index 3641a97c..710148a8 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -31,12 +31,11 @@ process MIRTRACE_RUN { def config_lines = [ids,reads] .transpose() .collect({ id, path -> path.getFileName().toString()}) + .map({path -> "echo ./${path}\n >> mirtrace_config"}) """ export mirtracejar=\$(dirname \$(which mirtrace)) - ${config_lines.join("\n ")} - java $java_mem -jar \$mirtracejar/mirtrace.jar --mirtrace-wrapper-name mirtrace qc \\ --species $params.mirtrace_species \\ $primer \\ From d39200cf3902609d3c1b680b57b6be3f3e349502 Mon Sep 17 00:00:00 2001 From: Felipe Marques de Almeida Date: Wed, 3 Apr 2024 08:49:59 +0000 Subject: [PATCH 14/32] perform operation outside module --- modules/local/mirtrace.nf | 10 ++-------- subworkflows/local/mirtrace.nf | 13 ++++++++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index 710148a8..d9b5eb39 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -8,6 +8,7 @@ process MIRTRACE_RUN { input: tuple val(adapter), val(ids), path(reads) + path(mirtrace_config) output: path "mirtrace/*" , emit: mirtrace @@ -26,13 +27,6 @@ process MIRTRACE_RUN { java_mem = "-Xms${tmem} -Xmx${tmem}" } - //Staging the files as path() but then getting the filenames for the config file that mirtrace needs - //Directly using val(reads) as in previous versions is not reliable as staging between work directories is not 100% reliable if not explicitly defined via nextflow itself - def config_lines = [ids,reads] - .transpose() - .collect({ id, path -> path.getFileName().toString()}) - .map({path -> "echo ./${path}\n >> mirtrace_config"}) - """ export mirtracejar=\$(dirname \$(which mirtrace)) @@ -40,7 +34,7 @@ process MIRTRACE_RUN { --species $params.mirtrace_species \\ $primer \\ $protocol \\ - --config mirtrace_config \\ + --config $mirtrace_config \\ --write-fasta \\ --output-dir mirtrace \\ --force diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index 317c4444..d9dea8b6 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -9,7 +9,18 @@ workflow MIRTRACE { reads // channel: [ val(adapterseq), [ val(ids) ], [ path(reads) ] ] main: - reads | MIRTRACE_RUN + + //Staging the files as path() but then getting the filenames for the config file that mirtrace needs + //Directly using val(reads) as in previous versions is not reliable as staging between work directories is not 100% reliable if not explicitly defined via nextflow itself + ch_mirtrace_config = + reads.map { adapter, ids, reads -> [ids,reads]} + .transpose() + .collectFile { id, path -> "./${path.getFileName().toString()}\n" } // operations need a channel, so, should be outside the module + + MIRTRACE_RUN ( + reads, + ch_mirtrace_config + ) emit: results = MIRTRACE_RUN.out.mirtrace From 6f4b3272fab723789e73121c18bc1c727cd91a5a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 3 Apr 2024 08:54:16 +0000 Subject: [PATCH 15/32] Fix for casting bug --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b50098..7110da62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## v2.3.1 - XXXX-XX-XX- Gray Zinc Dalmation Patch - [[#328]](https://github.com/nf-core/smrnaseq/pull/328) - Fix [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module -- [[#331]](https://github.com/nf-core/smrnaseq/pull/331) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module +- [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module ### Software dependencies From 3824d680df35cb4e5d8444b4a8c304ce3a871456 Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Thu, 4 Apr 2024 08:19:50 +0000 Subject: [PATCH 16/32] accept auto-detect as parameter value to activate adapter auto detection --- CHANGELOG.md | 1 + conf/modules.config | 12 ++++++------ docs/usage.md | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d8f261b..7e547293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [input channel cardinality](https://github.com/nf-core/smrnaseq/issues/331) in `MIRDEEP2_RUN` module - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [bowtie conda version](https://github.com/nf-core/smrnaseq/issues/333) in `BOWTIE_MAP_SEQ` module - [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module +- [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/327), allow `auto-detect` as value ### Software dependencies diff --git a/conf/modules.config b/conf/modules.config index f7a50622..4130afa7 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -47,12 +47,12 @@ process { // withName: '.*:FASTQ_FASTQC_UMITOOLS_FASTP:FASTP' { ext.args = [ "", - params.trim_fastq ? "" : "--disable_adapter_trimming", - params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. - params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. - params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", - params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" + params.trim_fastq ? "" : "--disable_adapter_trimming", + params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. + params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. + params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", + params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", + params.three_prime_adapter == "auto-detect" ? "" : "--adapter_sequence ${params.three_prime_adapter}" ].join(" ").trim() publishDir = [ [ diff --git a/docs/usage.md b/docs/usage.md index b5eb01f0..881bb2ff 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -18,7 +18,7 @@ This option indicates the experimental protocol used for the sample preparation. The parameter `--three_prime_adapter` is set to the Illumina TruSeq single index adapter sequence `AGATCGGAAGAGCACACGTCTGAACTCCAGTCA`. This is also to ensure, that the auto-detect functionality of `FASTP` is disabled. Please make sure to adapt this adapter sequence accordingly for your run. -:warning: At least the `custom` protocol has to be specified, otherwise the pipeline won't run. In case you specify the `custom` protocol, ensure that the parameters above are set accordingly or the defaults will be applied. If you want to auto-detect the adapters using `fastp`, please set `--three_prime_adapter` to `""`. +:warning: At least the `custom` protocol has to be specified, otherwise the pipeline won't run. In case you specify the `custom` protocol, ensure that the parameters above are set accordingly or the defaults will be applied. If you want to auto-detect the adapters using `fastp`, please set `--three_prime_adapter` to `auto-detect`. ### `mirtrace_species` or `mirgenedb_species` From 2bfbbd5a6936abc8c5ebfe22341f71b78f0a5460 Mon Sep 17 00:00:00 2001 From: Christopher Mohr Date: Thu, 4 Apr 2024 08:22:04 +0000 Subject: [PATCH 17/32] reference correct issue in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e547293..872a049d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [input channel cardinality](https://github.com/nf-core/smrnaseq/issues/331) in `MIRDEEP2_RUN` module - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [bowtie conda version](https://github.com/nf-core/smrnaseq/issues/333) in `BOWTIE_MAP_SEQ` module - [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module -- [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/327), allow `auto-detect` as value +- [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/326), allow `auto-detect` as value ### Software dependencies From 7a34574d06e0781c7160ec67cc999a1a6f3f421b Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:32:50 +0000 Subject: [PATCH 18/32] Attempt at fixing the mirtrace part for phred scores --- CHANGELOG.md | 1 + nextflow.config | 3 +++ nextflow_schema.json | 5 +++++ subworkflows/local/mirtrace.nf | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 872a049d..0d672f1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [bowtie conda version](https://github.com/nf-core/smrnaseq/issues/333) in `BOWTIE_MAP_SEQ` module - [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module - [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/326), allow `auto-detect` as value +- [[#342]](https://github.com/nf-core/smrnaseq/pull/342) - Fix [phred offset issue](https://github.com/nf-core/smrnaseq/issues/341), allow specifying phred offsets for files ### Software dependencies diff --git a/nextflow.config b/nextflow.config index 1ac5eaeb..5f2651a5 100644 --- a/nextflow.config +++ b/nextflow.config @@ -68,6 +68,9 @@ params { pirna = null other_contamination = null + //FASTQ handling defaults, for mirtrace + phred_offset = '33' + // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' diff --git a/nextflow_schema.json b/nextflow_schema.json index cf2d0382..42680715 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -272,6 +272,11 @@ "type": "boolean", "description": "Save merged reads.", "default": true + }, + "phred_offset": { + "type": "integer", + "default": 33, + "description": "The PHRED quality offset to be used for any input fastq files. Default is 33, standard Illumina 1.8+ format." } } }, diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index d9dea8b6..e47a69cc 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -15,7 +15,7 @@ workflow MIRTRACE { ch_mirtrace_config = reads.map { adapter, ids, reads -> [ids,reads]} .transpose() - .collectFile { id, path -> "./${path.getFileName().toString()}\n" } // operations need a channel, so, should be outside the module + .collectFile { id, path -> "./${path.getFileName().toString()},${params.phred_offset}\n" } // operations need a channel, so, should be outside the module MIRTRACE_RUN ( reads, From 1bf28d2653420493ffbf615584f5979549a3ac0a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:34:46 +0000 Subject: [PATCH 19/32] Improved wording --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d672f1d..a06376fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [bowtie conda version](https://github.com/nf-core/smrnaseq/issues/333) in `BOWTIE_MAP_SEQ` module - [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module - [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/326), allow `auto-detect` as value -- [[#342]](https://github.com/nf-core/smrnaseq/pull/342) - Fix [phred offset issue](https://github.com/nf-core/smrnaseq/issues/341), allow specifying phred offsets for files +- [[#342]](https://github.com/nf-core/smrnaseq/pull/342) - Fix [phred offset issue](https://github.com/nf-core/smrnaseq/issues/341), allow specifying phred offset for FASTQ files ### Software dependencies From 4eafcacef5b67e7bb83c9ba259ab8fe95e19fbc4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:36:09 +0000 Subject: [PATCH 20/32] a bit of docs --- docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/output.md b/docs/output.md index d2358724..10d3e677 100644 --- a/docs/output.md +++ b/docs/output.md @@ -153,7 +153,7 @@ MultiQC reports the number of reads that were removed by each of the contaminant ## miRTrace -[miRTrace](https://github.com/friedlanderlab/mirtrace) is a quality control specifically for small RNA sequencing data (smRNA-Seq). Each sample is characterized by profiling sequencing quality, read length, sequencing depth and miRNA complexity and also the amounts of miRNAs versus undesirable sequences (derived from tRNAs, rRNAs and sequencing artifacts). +[miRTrace](https://github.com/friedlanderlab/mirtrace) is a quality control specifically for small RNA sequencing data (smRNA-Seq). Each sample is characterized by profiling sequencing quality, read length, sequencing depth and miRNA complexity and also the amounts of miRNAs versus undesirable sequences (derived from tRNAs, rRNAs and sequencing artifacts). By default, the pipeline sets the PHRED-offset to the most common +33, so if you need to adjust this, use the `params.phred_offset` option to include this accordingly for your FASTQ files. **Output directory: `results/mirtrace`** From ab117d70dc227f7208a693d1acc45eaf0b9d117d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:39:00 +0000 Subject: [PATCH 21/32] Add comma from review --- subworkflows/local/mirtrace.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index e47a69cc..f1d68d5f 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -15,7 +15,7 @@ workflow MIRTRACE { ch_mirtrace_config = reads.map { adapter, ids, reads -> [ids,reads]} .transpose() - .collectFile { id, path -> "./${path.getFileName().toString()},${params.phred_offset}\n" } // operations need a channel, so, should be outside the module + .collectFile { id, path -> "./${path.getFileName().toString()},,${params.phred_offset}\n" } // operations need a channel, so, should be outside the module MIRTRACE_RUN ( reads, From 75e462b68805730d5b3967e55fad195337b6dd6f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:44:22 +0000 Subject: [PATCH 22/32] Fixed to match type --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 5f2651a5..a377a698 100644 --- a/nextflow.config +++ b/nextflow.config @@ -69,7 +69,7 @@ params { other_contamination = null //FASTQ handling defaults, for mirtrace - phred_offset = '33' + phred_offset = 33 // References genome = null From bb96ff1eca58773c1f7166de472469bea7f47544 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:49:10 +0000 Subject: [PATCH 23/32] Fix adding id too --- subworkflows/local/mirtrace.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index f1d68d5f..0d775a70 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -15,7 +15,7 @@ workflow MIRTRACE { ch_mirtrace_config = reads.map { adapter, ids, reads -> [ids,reads]} .transpose() - .collectFile { id, path -> "./${path.getFileName().toString()},,${params.phred_offset}\n" } // operations need a channel, so, should be outside the module + .collectFile { id, path -> "./${path.getFileName().toString()}",id,,"${params.phred_offset}\n" } // operations need a channel, so, should be outside the module MIRTRACE_RUN ( reads, From accc707adb6f3fe4f87c37bc53f387c83791ebb7 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 09:51:03 +0000 Subject: [PATCH 24/32] Fix addition of id --- subworkflows/local/mirtrace.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index 0d775a70..8a87e1c1 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -15,7 +15,7 @@ workflow MIRTRACE { ch_mirtrace_config = reads.map { adapter, ids, reads -> [ids,reads]} .transpose() - .collectFile { id, path -> "./${path.getFileName().toString()}",id,,"${params.phred_offset}\n" } // operations need a channel, so, should be outside the module + .collectFile { id, path -> "./${path.getFileName().toString()},${id},,${params.phred_offset}\n" } // operations need a channel, so, should be outside the module MIRTRACE_RUN ( reads, From 50de711229030a92c14513b01ce3156ca7bc8e8c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 10:04:55 +0000 Subject: [PATCH 25/32] Fix adapter not specified --- subworkflows/local/mirtrace.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index 8a87e1c1..b42b2977 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -12,10 +12,11 @@ workflow MIRTRACE { //Staging the files as path() but then getting the filenames for the config file that mirtrace needs //Directly using val(reads) as in previous versions is not reliable as staging between work directories is not 100% reliable if not explicitly defined via nextflow itself + //mirtrace is a bit peculiar in parsing these config files, so looked it up in the source how its done. this way should work ch_mirtrace_config = reads.map { adapter, ids, reads -> [ids,reads]} .transpose() - .collectFile { id, path -> "./${path.getFileName().toString()},${id},,${params.phred_offset}\n" } // operations need a channel, so, should be outside the module + .collectFile { id, path -> "./${path.getFileName().toString()},${id},${params.phred_offset}\n" } // operations need a channel, so, should be outside the module MIRTRACE_RUN ( reads, From ee857c5fbeb41f40086ed5d440d69615f75abf32 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 10:12:02 +0000 Subject: [PATCH 26/32] Local adapters, not globals anymore --- modules/local/mirtrace.nf | 2 -- subworkflows/local/mirtrace.nf | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index d9b5eb39..87526016 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -19,7 +19,6 @@ process MIRTRACE_RUN { script: // mirtrace protocol defaults to 'params.protocol' if not set - def primer = adapter ? "--adapter ${adapter}" : "" def protocol = params.protocol == 'custom' ? '' : "--protocol $params.protocol" def java_mem = '' if(task.memory){ @@ -32,7 +31,6 @@ process MIRTRACE_RUN { java $java_mem -jar \$mirtracejar/mirtrace.jar --mirtrace-wrapper-name mirtrace qc \\ --species $params.mirtrace_species \\ - $primer \\ $protocol \\ --config $mirtrace_config \\ --write-fasta \\ diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index b42b2977..3d2b6002 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -16,7 +16,7 @@ workflow MIRTRACE { ch_mirtrace_config = reads.map { adapter, ids, reads -> [ids,reads]} .transpose() - .collectFile { id, path -> "./${path.getFileName().toString()},${id},${params.phred_offset}\n" } // operations need a channel, so, should be outside the module + .collectFile { adapter, id, path -> "./${path.getFileName().toString()},${id},${adapter},${params.phred_offset}\n" } // operations need a channel, so, should be outside the module MIRTRACE_RUN ( reads, From aecb2188edabb99581ef283a9fdc80e18f776a0c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 17 Apr 2024 10:21:22 +0000 Subject: [PATCH 27/32] Fix passing on adapter seq to mirtrace module --- subworkflows/local/mirtrace.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index 3d2b6002..528e4233 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -14,7 +14,7 @@ workflow MIRTRACE { //Directly using val(reads) as in previous versions is not reliable as staging between work directories is not 100% reliable if not explicitly defined via nextflow itself //mirtrace is a bit peculiar in parsing these config files, so looked it up in the source how its done. this way should work ch_mirtrace_config = - reads.map { adapter, ids, reads -> [ids,reads]} + reads.map { adapter, ids, reads -> [adapter, ids,reads]} .transpose() .collectFile { adapter, id, path -> "./${path.getFileName().toString()},${id},${adapter},${params.phred_offset}\n" } // operations need a channel, so, should be outside the module From 6550f946e9d86595d500770c5b966b9d5d443411 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 18 Apr 2024 11:20:52 +0000 Subject: [PATCH 28/32] Fix for mirdeep2 --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 4130afa7..e67745fe 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -329,14 +329,14 @@ process { // // MIRDEEP // - withName: 'MIRDEEP2_MAPPER' { + withName: 'NFCORE_SMRNASEQ:MIRDEEP2:MIRDEEP2_MAPPER' { publishDir = [ path: { "${params.outdir}/mirdeep2/mapper" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'MIRDEEP2_RUN' { + withName: 'NFCORE_SMRNASEQ:MIRDEEP2:MIRDEEP2_RUN' { publishDir = [ path: { "${params.outdir}/mirdeep2/run" }, mode: params.publish_dir_mode, From 90d6b1636ba44b0335454cd45bdd320874db472a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 18 Apr 2024 11:20:57 +0000 Subject: [PATCH 29/32] Version bump --- CHANGELOG.md | 3 ++- assets/multiqc_config.yml | 4 ++-- nextflow.config | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a06376fe..7c911d94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.3.1 - XXXX-XX-XX- Gray Zinc Dalmation Patch +## v2.3.1 - 2024-04-18 - Gray Zinc Dalmation Patch - [[#328]](https://github.com/nf-core/smrnaseq/pull/328) - Fix [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module - [[#334]](https://github.com/nf-core/smrnaseq/pull/334) - Fix [input channel cardinality](https://github.com/nf-core/smrnaseq/issues/331) in `MIRDEEP2_RUN` module @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module - [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/326), allow `auto-detect` as value - [[#342]](https://github.com/nf-core/smrnaseq/pull/342) - Fix [phred offset issue](https://github.com/nf-core/smrnaseq/issues/341), allow specifying phred offset for FASTQ files +- [[#342]](https://github.com/nf-core/smrnaseq/pull/343) - Fix [mirdeep2 output missing](https://github.com/nf-core/smrnaseq/issues/330), fix mirdeep2 outputs missing in outdir ### Software dependencies diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 2b3c2bd9..9264f6fa 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/smrnaseq + This report has been generated by the nf-core/smrnaseq analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-smrnaseq-methods-description": diff --git a/nextflow.config b/nextflow.config index a377a698..147461f1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -308,7 +308,7 @@ manifest { description = """Small RNA-Seq Best Practice Analysis Pipeline.""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '2.3.1dev' + version = '2.3.1' doi = '10.5281/zenodo.3456879' } From 3a65e943061c3d5facaf9c5531808bdfe4211151 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 18 Apr 2024 13:29:28 +0200 Subject: [PATCH 30/32] Update CHANGELOG.md Co-authored-by: Christopher Mohr --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c911d94..c42d00e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#335]](https://github.com/nf-core/smrnaseq/pull/335) - Final fix for [casting issue](https://github.com/nf-core/smrnaseq/issues/327) in mirtrace module - [[#337]](https://github.com/nf-core/smrnaseq/pull/337) - Fix [three_prime_adapter issue](https://github.com/nf-core/smrnaseq/issues/326), allow `auto-detect` as value - [[#342]](https://github.com/nf-core/smrnaseq/pull/342) - Fix [phred offset issue](https://github.com/nf-core/smrnaseq/issues/341), allow specifying phred offset for FASTQ files -- [[#342]](https://github.com/nf-core/smrnaseq/pull/343) - Fix [mirdeep2 output missing](https://github.com/nf-core/smrnaseq/issues/330), fix mirdeep2 outputs missing in outdir +- [[#343]](https://github.com/nf-core/smrnaseq/pull/343) - Fix [mirdeep2 output missing](https://github.com/nf-core/smrnaseq/issues/330), fix mirdeep2 outputs missing in outdir ### Software dependencies From 09dadb7f553c4d3bab7378dc96acd57765610255 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 18 Apr 2024 11:54:48 +0000 Subject: [PATCH 31/32] Bump several modules to latest version --- modules.json | 10 +- modules/nf-core/fastp/main.nf | 2 +- modules/nf-core/fastp/tests/main.nf.test | 18 +- modules/nf-core/fastp/tests/main.nf.test.snap | 10 +- .../fastp/tests/nextflow.interleaved.config | 5 + ...low.config => nextflow.save_failed.config} | 3 +- modules/nf-core/fastqc/main.nf | 6 + .../nf-core/umicollapse/tests/main.nf.test | 14 +- .../umicollapse/tests/main.nf.test.snap | 8 +- .../umitools/extract/tests/main.nf.test | 8 +- .../tests/main.nf.test | 513 +++++++++++++++++- .../tests/main.nf.test.snap | 458 +++++++++++++++- .../tests/nextflow.config | 11 + .../tests/nextflow.save_trimmed.config | 6 + 14 files changed, 1006 insertions(+), 66 deletions(-) create mode 100644 modules/nf-core/fastp/tests/nextflow.interleaved.config rename modules/nf-core/fastp/tests/{nextflow.config => nextflow.save_failed.config} (50%) create mode 100644 subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config create mode 100644 subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config diff --git a/modules.json b/modules.json index 1d31a680..109997b3 100644 --- a/modules.json +++ b/modules.json @@ -17,12 +17,12 @@ }, "fastp": { "branch": "master", - "git_sha": "003920c7f9a8ae19b69a97171922880220bedf56", + "git_sha": "95cf5fe0194c7bf5cb0e3027a2eb7e7c89385080", "installed_by": ["fastq_fastqc_umitools_fastp", "modules"] }, "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["fastq_fastqc_umitools_fastp"] }, "multiqc": { @@ -57,12 +57,12 @@ }, "umicollapse": { "branch": "master", - "git_sha": "ba3f3df395d2719dcef5c67189042a1dc555c701", + "git_sha": "b97197968ac12dde2463fa54541f6350c46f2035", "installed_by": ["modules"] }, "umitools/extract": { "branch": "master", - "git_sha": "ff7e93715a2acecf3f143ec78c9858deba2984d0", + "git_sha": "d2c5e76f291379f3dd403e48e46ed7e6ba5da744", "installed_by": ["fastq_fastqc_umitools_fastp", "modules"] }, "untarfiles": { @@ -86,7 +86,7 @@ }, "fastq_fastqc_umitools_fastp": { "branch": "master", - "git_sha": "003920c7f9a8ae19b69a97171922880220bedf56", + "git_sha": "cabcc0dadf8366aa7a9930066a7b3dd90d9825d5", "installed_by": ["subworkflows"] }, "utils_nextflow_pipeline": { diff --git a/modules/nf-core/fastp/main.nf b/modules/nf-core/fastp/main.nf index 2a3b679e..4fc19b74 100644 --- a/modules/nf-core/fastp/main.nf +++ b/modules/nf-core/fastp/main.nf @@ -29,7 +29,7 @@ process FASTP { def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def adapter_list = adapter_fasta ? "--adapter_fasta ${adapter_fasta}" : "" - def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' + def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--failed_out ${prefix}.paired.fail.fastq.gz --unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' // Added soft-links to original fastqs for consistent naming in MultiQC // Use single ended for interleaved. Add --interleaved_in in config. if ( task.ext.args?.contains('--interleaved_in') ) { diff --git a/modules/nf-core/fastp/tests/main.nf.test b/modules/nf-core/fastp/tests/main.nf.test index 9b3f9a38..6f1f4897 100644 --- a/modules/nf-core/fastp/tests/main.nf.test +++ b/modules/nf-core/fastp/tests/main.nf.test @@ -251,7 +251,8 @@ nextflow_process { } test("fastp test_fastp_interleaved") { - config './nextflow.config' + + config './nextflow.interleaved.config' when { params { outdir = "$outputDir" @@ -277,7 +278,7 @@ nextflow_process { def html_text = [ "Q20 bases:25.719000 K (93.033098%)", "paired end (151 cycles + 151 cycles)"] def log_text = [ "Q20 bases: 12922(92.9841%)", - "reads passed filter: 198"] + "reads passed filter: 162"] def read_lines = [ "@ERR5069949.2151832 NS500628:121:HK3MMAFX2:2:21208:10793:15304/1", "TCATAAACCAAAGCACTCACAGTGTCAACAATTTCAGCAGGACAACGCCGACAAGTTCCGAGGAACATGTCTGGACCTATAGTTTTCATAAGTCTACACACTGAATTGAAATATTCTGGTTCTAGTGTGCCCTTAGTTAGCAATGTGCGT", "AAAAAAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAEEEEE - { assert path(process.out.reads_fail.get(0).get(1).get(1)).linesGzip.contains(failed_read2_line) } + { assert path(process.out.reads_fail.get(0).get(1).get(2)).linesGzip.contains(failed_read2_line) } } }, { html_text.each { html_part -> diff --git a/modules/nf-core/fastp/tests/main.nf.test.snap b/modules/nf-core/fastp/tests/main.nf.test.snap index b4c0e1dd..3e876288 100644 --- a/modules/nf-core/fastp/tests/main.nf.test.snap +++ b/modules/nf-core/fastp/tests/main.nf.test.snap @@ -7,7 +7,7 @@ "id": "test", "single_end": true }, - "test.fastp.json:md5,168f516f7bd4b7b6c32da7cba87299a4" + "test.fastp.json:md5,b24e0624df5cc0b11cd5ba21b726fb22" ] ] ], @@ -15,7 +15,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-17T18:08:06.123035" + "timestamp": "2024-03-18T16:19:15.063001" }, "test_fastp_paired_end_merged-for_stub_match": { "content": [ @@ -65,7 +65,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-17T18:06:00.223817" + "timestamp": "2024-03-18T16:18:43.526412" }, "versions_paired_end": { "content": [ @@ -112,7 +112,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-01T12:03:37.827323085" + "timestamp": "2024-03-18T16:19:15.111894" }, "test_fastp_paired_end_merged_match": { "content": [ @@ -283,7 +283,7 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-02-01T11:57:30.791982648" + "timestamp": "2024-03-18T16:18:43.580336" }, "versions_paired_end_merged_adapterlist": { "content": [ diff --git a/modules/nf-core/fastp/tests/nextflow.interleaved.config b/modules/nf-core/fastp/tests/nextflow.interleaved.config new file mode 100644 index 00000000..4be8dbd2 --- /dev/null +++ b/modules/nf-core/fastp/tests/nextflow.interleaved.config @@ -0,0 +1,5 @@ +process { + withName: FASTP { + ext.args = "--interleaved_in -e 30" + } +} diff --git a/modules/nf-core/fastp/tests/nextflow.config b/modules/nf-core/fastp/tests/nextflow.save_failed.config similarity index 50% rename from modules/nf-core/fastp/tests/nextflow.config rename to modules/nf-core/fastp/tests/nextflow.save_failed.config index 0f7849ad..53b61b0c 100644 --- a/modules/nf-core/fastp/tests/nextflow.config +++ b/modules/nf-core/fastp/tests/nextflow.save_failed.config @@ -1,6 +1,5 @@ process { - withName: FASTP { - ext.args = "--interleaved_in" + ext.args = "-e 30" } } diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 9e19a74c..d79f1c86 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/umicollapse/tests/main.nf.test b/modules/nf-core/umicollapse/tests/main.nf.test index 36cd748b..2dec45b2 100644 --- a/modules/nf-core/umicollapse/tests/main.nf.test +++ b/modules/nf-core/umicollapse/tests/main.nf.test @@ -34,10 +34,7 @@ nextflow_process { script "../../bwa/index/main.nf" process{ """ - input[0] = [ - [ id:'sarscov2'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] + input[0] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] """ } } @@ -47,7 +44,8 @@ nextflow_process { """ input[0] = UMITOOLS_EXTRACT.out.reads input[1] = BWA_INDEX.out.index - input[2] = true + input[2] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = true """ } } @@ -117,7 +115,8 @@ nextflow_process { """ input[0] = UMITOOLS_EXTRACT.out.reads input[1] = BWA_INDEX.out.index - input[2] = true + input[2] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = true """ } } @@ -214,7 +213,8 @@ nextflow_process { """ input[0] = UMITOOLS_EXTRACT.out.reads input[1] = BWA_INDEX.out.index - input[2] = true + input[2] = [[ id:'sarscov2'],file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)] + input[3] = true """ } } diff --git a/modules/nf-core/umicollapse/tests/main.nf.test.snap b/modules/nf-core/umicollapse/tests/main.nf.test.snap index 23916f0b..861e9ca6 100644 --- a/modules/nf-core/umicollapse/tests/main.nf.test.snap +++ b/modules/nf-core/umicollapse/tests/main.nf.test.snap @@ -16,9 +16,9 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T11:24:31.850566925" + "timestamp": "2024-03-14T13:41:23.869211282" }, "umicollapse fastq tests": { "content": [ @@ -117,8 +117,8 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-19T11:24:44.166029769" + "timestamp": "2024-03-14T13:41:54.486079388" } } \ No newline at end of file diff --git a/modules/nf-core/umitools/extract/tests/main.nf.test b/modules/nf-core/umitools/extract/tests/main.nf.test index 83a77a1b..2a8eba15 100644 --- a/modules/nf-core/umitools/extract/tests/main.nf.test +++ b/modules/nf-core/umitools/extract/tests/main.nf.test @@ -12,9 +12,6 @@ nextflow_process { test("Should run without failures") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ [ id:'test', single_end:true ], // meta map @@ -26,10 +23,9 @@ nextflow_process { then { assertAll ( - { assert process.success }, - { assert snapshot(process.out.versions).match("versions") } + { assert process.success }, + { assert snapshot(process.out.versions).match("versions") } ) } - } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test index 8f1d82d3..961b5b4f 100644 --- a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test @@ -3,6 +3,8 @@ nextflow_workflow { name "Test Workflow FASTQ_FASTQC_UMITOOLS_FASTP" script "../main.nf" workflow "FASTQ_FASTQC_UMITOOLS_FASTP" + config './nextflow.config' + tag "subworkflows" tag "subworkflows_nfcore" tag "subworkflows/fastq_fastqc_umitools_fastp" @@ -29,8 +31,10 @@ nextflow_workflow { input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true)] + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] ]) input[1] = skip_fastqc input[2] = with_umi @@ -48,14 +52,503 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(workflow.out.reads).match("reads") }, - { assert snapshot(workflow.out.umi_log).match("umi_log") }, - { assert snapshot(workflow.out.trim_json).match("trim_json") }, - { assert snapshot(workflow.out.trim_reads_fail).match("trim_reads_fail") }, - { assert snapshot(workflow.out.trim_reads_merged).match("trim_reads_merged") }, - { assert snapshot(workflow.out.adapter_seq).match("adapter_seq") }, - { assert snapshot(workflow.out.trim_read_count).match("trim_read_count") }, - { assert snapshot(workflow.out.versions).match("versions") }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("skip_fastqc") { + + when { + workflow { + """ + skip_fastqc = true + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end: false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert !workflow.out.fastqc_raw_html }, + { assert !workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert !workflow.out.fastqc_trim_html }, + { assert !workflow.out.fastqc_trim_zip } + ) + } + } + + test("with_umi") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = true + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + + test("skip_umi_extract") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = true + skip_umi_extract = true + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("umi_discard_read = 2") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = true + skip_umi_extract = true + umi_discard_read = 2 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("skip_trimming") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = true + adapter_fasta = [] + save_trimmed_fail = false + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads.get(0).get(0), // Reads meta map + // Because the input file is passed to the output file, we have to do check the filename only + file(workflow.out.reads.get(0).get(1).get(0)).name, + file(workflow.out.reads.get(0).get(1).get(1)).name, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert !workflow.out.trim_html }, + { assert !workflow.out.trim_log }, + { assert !workflow.out.fastqc_trim_html }, + { assert !workflow.out.fastqc_trim_zip } + ) + } + } + + test("save_trimmed_fail") { + + config './nextflow.save_trimmed.config' + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = true + save_merged = false + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("save_merged") { + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = true + min_trimmed_reads = 1 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, + + { assert workflow.out.fastqc_raw_html }, + { assert workflow.out.fastqc_raw_zip }, + { assert workflow.out.trim_html }, + { assert workflow.out.trim_log }, + { assert workflow.out.fastqc_trim_html }, + { assert workflow.out.fastqc_trim_zip } + ) + } + } + + test("min_trimmed_reads = 26") { + // Subworkflow should stop after FASTP which trims down to 25 reads + + when { + workflow { + """ + skip_fastqc = false + with_umi = false + skip_umi_extract = false + umi_discard_read = 1 + skip_trimming = false + adapter_fasta = [] + save_trimmed_fail = false + save_merged = true + min_trimmed_reads = 26 + + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = skip_fastqc + input[2] = with_umi + input[3] = skip_umi_extract + input[4] = umi_discard_read + input[5] = skip_trimming + input[6] = adapter_fasta + input[7] = save_trimmed_fail + input[8] = save_merged + input[9] = min_trimmed_reads + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.reads, + workflow.out.umi_log, + workflow.out.trim_json, + workflow.out.trim_reads_fail, + workflow.out.trim_reads_merged, + workflow.out.adapter_seq, + workflow.out.trim_read_count, + workflow.out.versions + ).match() + }, { assert workflow.out.fastqc_raw_html }, { assert workflow.out.fastqc_raw_zip }, diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap index 89ba8da1..3e11d9ec 100644 --- a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/main.nf.test.snap @@ -1,30 +1,186 @@ { - "trim_reads_merged": { + "skip_fastqc": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,67b2bbae47f073e05a97a9c2edce23c7", + "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" + ] + ], [ + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.041635573" + "timestamp": "2024-03-18T16:53:49.315194" }, - "trim_reads_fail": { + "save_trimmed_fail": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,6ff32a64c5188b9a9192be1398c262c7", + "test_2.fastp.fastq.gz:md5,db0cb7c9977e94ac2b4b446ebd017a8a" + ] + ] + ], [ + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,4c3268ddb50ea5b33125984776aa3519" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test.paired.fail.fastq.gz:md5,409b687c734cedd7a1fec14d316e1366", + "test_1.fail.fastq.gz:md5,4f273cf3159c13f79e8ffae12f5661f6", + "test_2.fail.fastq.gz:md5,f97b9edefb5649aab661fbc9e71fc995" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 162 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.033284693" + "timestamp": "2024-03-18T16:51:45.34934" }, - "versions": { + "skip_umi_extract": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,67b2bbae47f073e05a97a9c2edce23c7", + "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" + ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], [ "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", @@ -35,10 +191,25 @@ "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.121510557" + "timestamp": "2024-03-18T12:07:40.34249" }, - "trim_json": { + "umi_discard_read = 2": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,67b2bbae47f073e05a97a9c2edce23c7", + "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" + ] + ] + ], + [ + + ], [ [ { @@ -47,16 +218,81 @@ }, "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.024410724" + "timestamp": "2024-03-18T12:08:24.141938" }, - "adapter_seq": { + "save_merged": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,54b726a55e992a869fd3fa778afe1672", + "test_2.fastp.fastq.gz:md5,29d3b33b869f7b63417b8ff07bb128ba" + ] + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,b712fd68ed0322f4bec49ff2a5237fcc" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.merged.fastq.gz:md5,c873bb1ab3fa859dcc47306465e749d5" + ] + ], [ [ { @@ -65,15 +301,65 @@ }, "unspecified" ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 75 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.08674429" + "timestamp": "2024-03-18T12:10:18.546963" }, - "reads": { + "skip_trimming": { + "content": [ + { + "id": "test", + "single_end": false + }, + "test_1.fastq.gz", + "test_2.fastq.gz", + [ + + ], + [ + + ], + [ + + ], + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T15:49:26.574759" + }, + "sarscov2 paired-end [fastq]": { "content": [ [ [ @@ -86,42 +372,178 @@ "test_2.fastp.fastq.gz:md5,25cbdca08e2083dbd4f0502de6b62f39" ] ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,1e0f8e27e71728e2b63fc64086be95cd" + ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 198 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:49.994419936" + "timestamp": "2024-03-18T16:53:39.139038" }, - "umi_log": { + "min_trimmed_reads = 26": { "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1.fastp.fastq.gz:md5,54b726a55e992a869fd3fa778afe1672", + "test_2.fastp.fastq.gz:md5,29d3b33b869f7b63417b8ff07bb128ba" + ] + ] + ], [ + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.fastp.json:md5,b712fd68ed0322f4bec49ff2a5237fcc" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test.merged.fastq.gz:md5,c873bb1ab3fa859dcc47306465e749d5" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "unspecified" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + 75 + ] + ], + [ + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.017720214" + "timestamp": "2024-03-18T11:52:23.849945" }, - "trim_read_count": { + "with_umi": { "content": [ [ [ { "id": "test", - "single_end": false + "single_end": true }, - 198 + "test.fastp.fastq.gz:md5,ba8c6c3a7ce718d9a2c5857e2edf53bc" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.fastp.json:md5,d39c5c6d9a2e35fb60d26ced46569af6" ] + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + 99 + ] + ], + [ + "versions.yml:md5,01f264f78de3c6d893c449cc6d3cd721", + "versions.yml:md5,85bd0117e5778fff18e3920972a296ad", + "versions.yml:md5,c50aa59475ab901bc6f9a2cf7b1a14e0", + "versions.yml:md5,f3dcaae948e8eed92b4a5557b4c6668e" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-12T08:38:50.102326089" + "timestamp": "2024-03-18T17:31:09.193212" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config new file mode 100644 index 00000000..12f7b257 --- /dev/null +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.config @@ -0,0 +1,11 @@ +process { + + withName: UMITOOLS_EXTRACT { + ext.args = '--bc-pattern="NNNN" --bc-pattern2="NNNN"' + } + + withName: UMICOLLAPSE { + ext.prefix = { "${meta.id}.dedup" } + } + +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config new file mode 100644 index 00000000..2430e9d5 --- /dev/null +++ b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/tests/nextflow.save_trimmed.config @@ -0,0 +1,6 @@ +process { + // Make filtering more aggressive to make more reads fail + withName: FASTP { + ext.args = "-e 30" + } +} \ No newline at end of file From 6039accae68e28d280bca3fdc95c109d2ac8fffe Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 18 Apr 2024 12:02:03 +0000 Subject: [PATCH 32/32] Add proper zenodo URI --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50e94346..ccb136f0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![GitHub Actions CI Status](https://github.com/nf-core/smrnaseq/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/ci.yml) -[![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/smrnaseq/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.10696391?labelColor=000000)](https://doi.org/10.5281/zenodo.10696391) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/)