Merge pull request #198 from maxulysse/only_refactor

remove all the nf-test logic from the refactor branch

.gitattributes

@@ -1,5 +1,4 @@
 *.config linguist-language=nextflow
 *.nf.test linguist-language=nextflow
-tests/**/*nf.test.snap linguist-generated
 modules/nf-core/** linguist-generated
 subworkflows/nf-core/** linguist-generated

.github/CONTRIBUTING.md

@@ -116,4 +116,3 @@ To get started:
 Devcontainer specs:
 
 - [DevContainer config](.devcontainer/devcontainer.json)
-- [Dockerfile](.devcontainer/Dockerfile)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -42,7 +42,7 @@ body:
     attributes:
       label: System information
       description: |
-        * Nextflow version _(eg. 22.10.1)_
+        * Nextflow version _(eg. 23.04.0)_
         * Hardware _(eg. HPC, Desktop, Cloud)_
         * Executor _(eg. slurm, local, awsbatch)_
         * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_

.github/workflows/ci.yml

@@ -1,67 +1,28 @@
-# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
   pull_request:
   release:
     types: [published]
-  merge_group:
-    types:
-      - checks_requested
-    branches:
-      - master
-      - dev
-      - refactor
 
 env:
   NXF_ANSI_LOG: false
-  NFTEST_VER: "0.8.0-rc1"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
-  changes:
-    name: Check for changes
-    runs-on: ubuntu-latest
-    outputs:
-      # Expose matched filters as job 'tags' output variable
-      tags: ${{ steps.filter.outputs.changes }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: ".github/tags.yml"
-
-  define_nxf_versions:
-    name: Choose nextflow versions to test against depending on target branch
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.nxf_versions.outputs.matrix }}
-    steps:
-      - id: nxf_versions
-        run: |
-          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
-            echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
-          else
-            echo matrix='["latest-everything", "22.10.1"]' | tee -a $GITHUB_OUTPUT
-          fi
-
   test:
-    name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
-    needs: [changes, define_nxf_versions]
-    if: needs.changes.outputs.tags != '[]'
+    name: Run pipeline with test data
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/fetchngs') }}"
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
-        NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
-        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
-        profile:
-          - "docker"
-
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
@@ -71,23 +32,29 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Install nf-test
+      - name: Run pipeline with test data
         run: |
-          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
-          sudo mv nf-test /usr/local/bin/
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
 
-      - name: Run nf-test
-        run: |
-          nf-test test --tag ${{ matrix.tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
+  parameters:
+    name: Test workflow parameters
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/fetchngs') }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        parameters:
+          - "--nf_core_pipeline rnaseq"
+          - "--ena_metadata_fields run_accession,experiment_accession,library_layout,fastq_ftp,fastq_md5 --sample_mapping_fields run_accession,library_layout"
+          - "--skip_fastq_download"
+          - "--force_sratools_download"
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
 
-      - name: Output log on failure
-        if: failure()
+      - name: Install Nextflow
         run: |
-          sudo apt install bat > /dev/null
-          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
-
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v3
-        if: always() # always run even if the previous step fails
-        with:
-          report_paths: test.xml
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+      - name: Run pipeline with various parameters
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results ${{ matrix.parameters }}

.gitignore

@@ -6,6 +6,3 @@ results/
 testing/
 testing*
 *.pyc
-.nf-test/
-nf-test
-.nf-test.log

.gitpod.yml

@@ -1,4 +1,9 @@
 image: nfcore/gitpod:latest
+tasks:
+  - name: Update Nextflow and setup pre-commit
+    command: |
+      pre-commit install --install-hooks
+      nextflow self-update
 
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack

.nf-core.yml

@@ -4,14 +4,5 @@ lint:
   files_exist:
     - conf/modules.config
   files_unchanged:
-    - .github/CONTRIBUTING.md
-    - .github/ISSUE_TEMPLATE/bug_report.yml
-    - assets/nf-core-fetchngs_logo_light.png
     - assets/sendmail_template.txt
     - lib/NfcoreTemplate.groovy
-    - .gitattributes
-  multiqc_config: false
-  nextflow_config:
-    - "params.validationShowHiddenParams"
-    - "params.validationSchemaIgnoreParams"
-  schema_params: false

CHANGELOG.md

@@ -3,23 +3,7 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [[1.10.1](https://github.com/nf-core/fetchngs/releases/tag/1.10.1)] - 2023-06-29
-
-### Credits
-
-Special thanks to the following for their contributions to the release:
-
-- [Sateesh Peri](https://github.com/sateeshperi)
-- [Edmund Miller](https://github.com/Emiller88)
-- [Maxime Garcia](https://github.com/maxulysse)
-
-Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
-
-### Enhancements & fixes
-
-- [#166](https://github.com/nf-core/fetchngs/issues/175) - Add NF-TEST pipeline end-to-end tests for existing CI tests
-- [#184](https://github.com/nf-core/fetchngs/pull/184) Emits all files in workflow for testing and enabling workflow to be imported (not supported by Nextflow)
-- [#195](https://github.com/nf-core/fetchngs/pull/195) Separate configuration files for all modules
+## dev - [date]
 
 ## [[1.10.0](https://github.com/nf-core/fetchngs/releases/tag/1.10.0)] - 2023-05-16
 

CITATIONS.md

@@ -57,5 +57,8 @@
 
 - [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241)
 
+  > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241.
+
 - [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/)
+
   > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675.

README.md

@@ -2,7 +2,7 @@
 
 [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
@@ -50,7 +50,7 @@ nextflow run nf-core/fetchngs \
 > provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
 > see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
 
-For more details, please refer to the [usage documentation](https://nf-co.re/fetchngs/usage) and the [parameter documentation](https://nf-co.re/fetchngs/parameters).
+For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/fetchngs/usage) and the [parameter documentation](https://nf-co.re/fetchngs/parameters).
 
 ## Supported ids
 

assets/methods_description_template.yml

@@ -3,17 +3,21 @@ description: "Suggested text and references to use when describing pipeline usag
 section_name: "nf-core/fetchngs Methods Description"
 section_href: "https://github.com/nf-core/fetchngs"
 plot_type: "html"
-## TODO nf-core: Update the HTML below to your prefered methods description, e.g. add publication citation for this pipeline
+## TODO nf-core: Update the HTML below to your preferred methods description, e.g. add publication citation for this pipeline
 ## You inject any metadata in the Nextflow '${workflow}' object
 data: |
   <h4>Methods</h4>
-  <p>Data was processed using nf-core/fetchngs v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (<a href="https://doi.org/10.1038/s41587-020-0439-x">Ewels <em>et al.</em>, 2020</a>).</p>
+  <p>Data was processed using nf-core/fetchngs v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (<a href="https://doi.org/10.1038/s41587-020-0439-x">Ewels <em>et al.</em>, 2020</a>), utilising reproducible software environments from the Bioconda (<a href="https://doi.org/10.1038/s41592-018-0046-7">Grüning <em>et al.</em>, 2018</a>) and Biocontainers (<a href="https://doi.org/10.1093/bioinformatics/btx192">da Veiga Leprevost <em>et al.</em>, 2017</a>) projects.</p>
   <p>The pipeline was executed with Nextflow v${workflow.nextflow.version} (<a href="https://doi.org/10.1038/nbt.3820">Di Tommaso <em>et al.</em>, 2017</a>) with the following command:</p>
   <pre><code>${workflow.commandLine}</code></pre>
+  <p>${tool_citations}</p>
   <h4>References</h4>
   <ul>
-    <li>Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. <a href="https://doi.org/10.1038/nbt.3820">https://doi.org/10.1038/nbt.3820</a></li>
-    <li>Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. <a href="https://doi.org/10.1038/s41587-020-0439-x">https://doi.org/10.1038/s41587-020-0439-x</a></li>
+    <li>Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. doi: <a href="https://doi.org/10.1038/nbt.3820">10.1038/nbt.3820</a></li>
+    <li>Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. doi: <a href="https://doi.org/10.1038/s41587-020-0439-x">10.1038/s41587-020-0439-x</a></li>
+    <li>Grüning, B., Dale, R., Sjödin, A., Chapman, B. A., Rowe, J., Tomkins-Tinch, C. H., Valieris, R., Köster, J., & Bioconda Team. (2018). Bioconda: sustainable and comprehensive software distribution for the life sciences. Nature Methods, 15(7), 475–476. doi: <a href="https://doi.org/10.1038/s41592-018-0046-7">10.1038/s41592-018-0046-7</a></li>
+    <li>da Veiga Leprevost, F., Grüning, B. A., Alves Aflitos, S., Röst, H. L., Uszkoreit, J., Barsnes, H., Vaudel, M., Moreno, P., Gatto, L., Weber, J., Bai, M., Jimenez, R. C., Sachsenberg, T., Pfeuffer, J., Vera Alvarez, R., Griss, J., Nesvizhskii, A. I., & Perez-Riverol, Y. (2017). BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics (Oxford, England), 33(16), 2580–2582. doi: <a href="https://doi.org/10.1093/bioinformatics/btx192">10.1093/bioinformatics/btx192</a></li>
+    ${tool_bibliography}
   </ul>
   <div class="alert alert-info">
     <h5>Notes:</h5>

assets/multiqc_config.yml

@@ -1,7 +1,7 @@
 report_comment: >
-  This report has been generated by the <a href="https://github.com/nf-core/fetchngs" target="_blank">nf-core/fetchngs</a>
+  This report has been generated by the <a href="https://github.com/nf-core/fetchngs/1.10.1dev" target="_blank">nf-core/fetchngs</a>
   analysis pipeline. For information about how to interpret these results, please see the
-  <a href="https://nf-co.re/fetchngs" target="_blank">documentation</a>.
+  <a href="https://nf-co.re/fetchngs/1.10.1dev/output" target="_blank">documentation</a>.
 report_section_order:
   "nf-core-fetchngs-methods-description":
     order: -1000

assets/schema_input.json

@@ -5,11 +5,13 @@
     "description": "Schema for the file provided with params.input",
     "type": "array",
     "items": {
-        "type": "array",
-        "items": {
-            "type": "string",
-            "pattern": "^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM])|(syn))(\\d+)$",
-            "errorMessage": "Please provide a valid SRA, ENA, DDBJ or GEO identifier"
+        "type": "object",
+        "properties": {
+            "id": {
+                "type": "string",
+                "pattern": "^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM])|(syn))(\\d+)$",
+                "errorMessage": "Please provide a valid SRA, ENA, DDBJ or GEO identifier"
+            }
         }
     }
 }

assets/slackreport.json

@@ -3,7 +3,7 @@
         {
             "fallback": "Plain-text summary of the attachment.",
             "color": "<% if (success) { %>good<% } else { %>danger<%} %>",
-            "author_name": "sanger-tol/readmapping v${version} - ${runName}",
+            "author_name": "nf-core/fetchngs v${version} - ${runName}",
             "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico",
             "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>",
             "fields": [

bin/sra_runinfo_to_ftp.py

@@ -61,8 +61,7 @@ def parse_sra_runinfo(file_in):
     with open(file_in, "r", newline="") as fin:
         reader = csv.DictReader(fin, delimiter="\t", skipinitialspace=True)
         header = list(reader.fieldnames)
-        missing = frozenset(columns).difference(frozenset(header))
-        if missing:
+        if missing := frozenset(columns).difference(frozenset(header)):
             logger.critical(f"The following expected columns are missing from {file_in}: " f"{', '.join(missing)}.")
             sys.exit(1)
         for row in reader:

conf/test.config

@@ -21,6 +21,4 @@ params {
 
     // Input data
     input  = 'https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs/sra_ids_test.csv'
-
-    schema_ignore_params = 'fastq_ftp_ids,genomes,merge_samplesheet_ids,test_data,test_data_base'
 }

conf/test_full.config

@@ -10,8 +10,6 @@
 ----------------------------------------------------------------------------------------
 */
 
-cleanup = true
-
 params {
     config_profile_name        = 'Full test profile'
     config_profile_description = 'Full test dataset to check pipeline function'

docs/usage.md

@@ -70,12 +70,6 @@ From v1.9 of this pipeline the default `strandedness` in the output samplesheet
 
 If FTP connections are blocked on your network use the [`--force_sratools_download`](https://nf-co.re/fetchngs/parameters#force_sratools_download) parameter to force the pipeline to download data using sra-tools instead of the ENA FTP.
 
-### Downloading 10X Genomics Data
-
-As of v1.10.0, this pipeline now supports downloading of 10X Genomics data. For 10X data sets, the output needs to be split into three fastq files, consisting of the feature barcode+UMI, cDNA sequence reads, and sample index. Adding the `--force_sratools_download` parameter will cause the pipeline to use the sra-tools fasterq-dump program with appropriate setting to ensure that all three files are generated.
-
-Please note that in our experience the downloaded R1, R2, and R3 fastq files do not always correspond to the same data types across different submissions. In many cases, R1 will be the cell barcode + UMI, R2 is the cDNA sequence, and R3 is the sample index. However, this is not alway the case. Users are strongly encouraged to verify their data before proceeding with downstream analysis.
-
 ### Downloading dbGAP data with JWT
 
 As of v1.10.0, the SRA Toolkit used in this pipeline can be configured to access protected data from dbGAP using a [JWT cart file](https://www.ncbi.nlm.nih.gov/sra/docs/sra-dbGAP-cloud-download/) on a supported cloud computing environment (Amazon Web Services or Google Cloud Platform). The JWT cart file can be specified with `--dbgap_key /path/to/cart.jwt`.