Merge pull request #412 from nschcolnicov/nextflex

Updating nextflex branch
nf-core · Sep 6, 2024 · 1a8a68e · 1a8a68e
2 parents bc228d3 + 669f7e1
commit 1a8a68e
Show file tree

Hide file tree

Showing 66 changed files with 3,068 additions and 423 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -4,12 +4,20 @@ on:
   push:
     branches:
       - dev
+      - master
   pull_request:
+    branches:
+      - dev
+      - master
   release:
     types: [published]
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.9.0"
+  NFT_WORKDIR: "~"
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
@@ -24,28 +32,49 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        shard: [1, 2, 3, 4]
         NXF_VER:
           - "23.04.0"
-          - "latest-everything"
-        profile:
-          - "test"
-          - "test_no_genome"
-          - "test_umi"
-          - "test_index"
-          - "test_technical_repeats"
-          - "test_mirgenedb"
+        profile: ["docker"]
+    env:
+      SHARDS: "4"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
 
-      - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v2
+      - name: Install pdiff to see diff between nf-test snapshots
+        run: |
+          python -m pip install --upgrade pip
+          pip install pdiff
+
+      - uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Disk space cleanup
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+      - uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
 
-      - name: Run pipeline with test data
+      - name: Run Tests (Shard ${{ matrix.shard }}/${{ env.SHARDS }})
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker --outdir ./results
+          nf-test test \
+              --ci \
+              --shard ${{ matrix.shard }}/${{ env.SHARDS }} \
+              --changed-since HEAD^ \
+              --profile "+${{ matrix.profile }}" \
+              --filter pipeline \
+              --junitxml=test.xml
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml
+          annotate_only: true
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ testing/
 testing*
 *.pyc
 execution_trace*
+.nf-test*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,19 +5,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## v2.4.0dev - 2024-XX-XX - X
 
-- [[#365]](https://github.com/nf-core/smrnaseq/issues/365) by [[#386]](https://github.com/nf-core/smrnaseq/pull/386). Fix Nextflex trimming support.
-- [[#332]](https://github.com/nf-core/smrnaseq/issues/332) by [[#361]](https://github.com/nf-core/smrnaseq/pull/361) - Fix documentation to use only single-end
-- [[#349]](https://github.com/nf-core/smrnaseq/pull/349) - Fix [MIRTOP_QUANT conda issue](https://github.com/nf-core/smrnaseq/issues/347), change conda-base to conda-forge channel
-- [[#350]](https://github.com/nf-core/smrnaseq/pull/350) - Fix [MIRTOP_QUANT conda issue](https://github.com/nf-core/smrnaseq/issues/347), set python version to 3.7 to fix pysam issue
-- [[#351]](https://github.com/nf-core/smrnaseq/issues/351) - Fix [Protocol inheritance issue](https://github.com/nf-core/smrnaseq/pull/364) - fixing protocol inheritance from subworkflow with move to config profile(s) for different protocols.
-- [[#369]](https://github.com/nf-core/smrnaseq/issues/369) - Fix [Linting](https://github.com/nf-core/smrnaseq/pull/377) - Fixed linting warnings and updated modules & subworkflows.
+- [[#349]](https://github.com/nf-core/smrnaseq/pull/349) - Fix [MIRTOP_QUANT conda issue](https://github.com/nf-core/smrnaseq/issues/347) - change conda-base to conda-forge channel.
+- [[#350]](https://github.com/nf-core/smrnaseq/pull/350) - Fix [MIRTOP_QUANT conda issue](https://github.com/nf-core/smrnaseq/issues/347) - set python version to 3.7 to fix pysam issue.
+- [[#361]](https://github.com/nf-core/smrnaseq/pull/361) - Fix [[#332]](https://github.com/nf-core/smrnaseq/issues/332) - Fix documentation to use only single-end.
+- [[#364]](https://github.com/nf-core/smrnaseq/pull/364) - Fix [Protocol inheritance issue](https://github.com/nf-core/smrnaseq/issues/351) - fixing protocol inheritance from subworkflow with move to config profile(s) for different protocols.
+- [[#372]](https://github.com/nf-core/smrnaseq/pull/372) - Fix [Plain test profile](https://github.com/nf-core/smrnaseq/issues/371) - Updated default protocol value to "custom".
 - [[#374]](https://github.com/nf-core/smrnaseq/pull/374) - Fix [default tests](https://github.com/nf-core/smrnaseq/issues/375) so that they do not require additional profiles in CI. Change GitHub CI fail-fast strategy to false.
-- [[#371]](https://github.com/nf-core/smrnaseq/issues/371) - Fix [Plain test profile](https://github.com/nf-core/smrnaseq/pull/372) - Updated default protocol value to "custom".
-- [[#380]](https://github.com/nf-core/smrnaseq/pull/380) - Fix checking number of samples which causes error in plotMDS. Add nf-tests for local modules using custom R scripts: [edgeR_mirBase.R](https://github.com/nf-core/smrnaseq/issues/187).
-- [[#378]](https://github.com/nf-core/smrnaseq/pull/378) - Fix [`--mirtrace_species` bug](<(https://github.com/nf-core/smrnaseq/issues/348)>). Make `MIRTRACE` process conditional. Add mirgenedb test.
-- [[#375]](https://github.com/nf-core/smrnaseq/pull/375) - Test merging of [technical repeats](https://github.com/nf-core/smrnaseq/issues/212).
-- [[#382]](https://github.com/nf-core/smrnaseq/pull/382) - Add nf-tests for local modules using custom R scripts: [collapse_mirtop.R](https://github.com/nf-core/smrnaseq/issues/174).
-- [[#384]](https://github.com/nf-core/smrnaseq/pull/384) - Fix filter stats module and add filter contaminants test profile: [filter status bug fix](https://github.com/nf-core/smrnaseq/issues/360).
+- [[#375]](https://github.com/nf-core/smrnaseq/pull/375) - Test [technical repeats](https://github.com/nf-core/smrnaseq/issues/212) - Test merging of technical repeats.
+- [[#377]](https://github.com/nf-core/smrnaseq/pull/377) - Fix [Linting](https://github.com/nf-core/smrnaseq/issues/369) - Fixed linting warnings and updated modules & subworkflows.
+- [[#378]](https://github.com/nf-core/smrnaseq/pull/378) - Fix [`--mirtrace_species` bug](<(https://github.com/nf-core/smrnaseq/issues/348)>) - Make `MIRTRACE` process conditional. Add mirgenedb test.
+- [[#380]](https://github.com/nf-core/smrnaseq/pull/380) - Fix [edgeR_mirBase.R](https://github.com/nf-core/smrnaseq/issues/187) - Fix checking number of samples which causes error in plotMDS. Add nf-tests for local modules using custom R scripts.
+- [[#381]](https://github.com/nf-core/smrnaseq/pull/381) - Update [Convert tests to nf-tests](https://github.com/nf-core/smrnaseq/issues/379) - CI tests to nf-tests.
+- [[#382]](https://github.com/nf-core/smrnaseq/pull/382) - Add [collapse_mirtop.R](https://github.com/nf-core/smrnaseq/issues/174) - Add nf-tests for local modules using custom R scripts.
+- [[#383]](https://github.com/nf-core/smrnaseq/pull/383) - Fix [parameter `--skip_fastp` throws an error](https://github.com/nf-core/smrnaseq/issues/263) - Fix parameter --skip_fastp.
+- [[#384]](https://github.com/nf-core/smrnaseq/pull/384) - Fix [filter status bug fix](https://github.com/nf-core/smrnaseq/issues/360) - Fix filter stats module and add filter contaminants test profile.
+- [[#386]](https://github.com/nf-core/smrnaseq/pull/386) - Fix [Nextflex trimming support](https://github.com/nf-core/smrnaseq/issues/365) - Fix Nextflex trimming support.
+- [[#387]](https://github.com/nf-core/smrnaseq/pull/387) - Add [contaminant filter failure because the Docker image for BLAT cannot be pulled](https://github.com/nf-core/smrnaseq/issues/354) - Add nf-test to local module `blat_mirna` and fixes . Adds a small test profile to test contaminant filter results.
+- [[#388]](https://github.com/nf-core/smrnaseq/pull/388) - Fix [igenomes fix](https://github.com/nf-core/smrnaseq/issues/360) - Fix workflow scripts so that they can use igenome parameters.
+- [[#391]](https://github.com/nf-core/smrnaseq/pull/391) - Fix [error because of large chromosomes](https://github.com/nf-core/smrnaseq/issues/132) - Change `.bai` index for `.csi` index in `samtools_index` to fix .
+- [[#392]](https://github.com/nf-core/smrnaseq/pull/392) - Update [Reduce tests](https://github.com/nf-core/smrnaseq/issues/389) - Combine and optimize tests, and reduce samplesheets sizes.
+- [[#397]](https://github.com/nf-core/smrnaseq/pull/397) - Fix [contaminant filter failure because of the Docker image for BLAT](https://github.com/nf-core/smrnaseq/issues/354) - Improvements to contaminant filter subworkflow and replacement for nf-core modules.
+- [[#398]](https://github.com/nf-core/smrnaseq/pull/398) - Update [Input channels](https://github.com/nf-core/smrnaseq/issues/390) - Updated channel and params handling through workflows.
+- [[#405]](https://github.com/nf-core/smrnaseq/pull/405) - Fix [Umicollapse algo wrong set](https://github.com/nf-core/smrnaseq/issues/404) - Fix potential bug in Umicollapse (not effective as we do not allow PE data in smrnaseq - but for consistency)
 
 ## v2.3.1 - 2024-04-18 - Gray Zinc Dalmation Patch
 

diff --git a/conf/modules.config b/conf/modules.config
@@ -49,7 +49,6 @@ process {
         ext.args = [ "",
             params.trim_fastq                           ? "" : "--disable_adapter_trimming",
             params.clip_r1 > 0                          ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1.
-        //    params.three_prime_clip_r1 > 0              ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed.
             params.fastp_min_length > 0                 ? "-l ${params.fastp_min_length}" : "",
             params.fastp_max_length > 0                 ? "--max_len1 ${params.fastp_max_length}" : "",
             params.three_prime_adapter == "auto-detect" ?  "" : "--adapter_sequence ${params.three_prime_adapter}"
@@ -70,13 +69,20 @@ process {
                 mode: params.publish_dir_mode,
                 pattern: "*.fail.fastq.gz",
                 enabled: params.save_trimmed_fail
+            ],
+            [
+                path: { "${params.outdir}/fastp/fastq" },
+                mode: params.publish_dir_mode,
+                pattern: "*.fastp.fastq.gz",
+                enabled: params.save_merged
             ]
         ]
     }
     //
     // FASTQ_FASTQC_UMITOOLS_FASTP
     //
     withName: '.*:FASTP3' {
+        ext.prefix =  { "${meta.id}.fastp3" }
         ext.args = [ "",
             "--disable_adapter_trimming",
             "--disable_quality_filtering",
@@ -164,7 +170,7 @@ process {
     //
 
     withName: '.*:UMICOLLAPSE_FASTQ' {
-        ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--method ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" }
+        ext.args = { meta.single_end ? "--algo ${params.umitools_method} --two-pass" : "--algo ${params.umitools_method} --two-pass --paired --remove-unpaired --remove-chimeric" }
         ext.prefix = { "${meta.id}.umi_dedup.sorted" }
         publishDir = [
             path: { "${params.outdir}/umi_dedup/bam_deduplicated" },
@@ -219,6 +225,24 @@ process {
         ]
     }
 
+    withName: 'NFCORE_SMRNASEQ:CONTAMINANT_FILTER:BLAT.*' {
+        ext.args = '-out=blast8'
+        ext.prefix = {"${meta.id}_${meta2.id}"}
+        tag = {"${meta.id} ${meta2.id}"}
+        publishDir = [ enabled: false ]
+    }
+
+    withName: 'NFCORE_SMRNASEQ:CONTAMINANT_FILTER:GAWK.*' {
+        ext.prefix = {"significant_hits_${meta.id}"}
+        publishDir = [ enabled: false ]
+    }
+
+    withName: 'NFCORE_SMRNASEQ:CONTAMINANT_FILTER:SEQKIT_GREP.*' {
+        ext.prefix = {"filtered_${meta.id}"}
+        ext.args = '-v'
+        publishDir = [ enabled: false ]
+    }
+
     //
     // MIRNA_QUANT
     //
@@ -332,6 +356,11 @@ process {
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
+
+    withName: 'SAMTOOLS_INDEX' {
+        ext.args = '-c'
+    }
+
     withName: 'NFCORE_SMRNASEQ:GENOME_QUANT:BAM_SORT_STATS_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' {
         ext.prefix = { "${meta.id}.sorted" }
         publishDir = [

diff --git a/conf/test.config b/conf/test.config
@@ -23,13 +23,13 @@ params {
 
     input            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet.csv'
     fasta            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/genome.fa'
+    bowtie_index     = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/bowtie_index.tar.gz'
 
     mirtrace_species         = 'hsa'
     skip_mirdeep             = true
     save_merged              = false
     save_aligned_mirna_quant = false
 
-    cleanup                  = true //Otherwise tests dont run through properly.
 }
 
 // Include illumina config to run test without additional profiles

diff --git a/conf/test_no_genome.config → conf/test_contamination.config b/conf/test_no_genome.config → conf/test_contamination.config
@@ -1,32 +1,39 @@
 /*
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Nextflow config file for running minimal tests
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Defines input files and everything required to run a fast and simple pipeline test.
 
     Use as follows:
-        nextflow run nf-core/smrnaseq -profile test,<docker/singularity>
+        nextflow run nf-core/smrnaseq -profile test_contamination,<docker/singularity> --outdir <OUTDIR>
 
 ----------------------------------------------------------------------------------------
 */
 
 params {
     config_profile_name        = 'Test profile'
-    config_profile_description = 'Minimal test dataset to check pipeline function'
+    config_profile_description = 'Minimal test dataset to check pipeline function with contamination filter'
 
     // Limit resources so that this can run on GitHub Actions
     max_cpus   = 2
     max_memory = '6.GB'
     max_time   = '6.h'
 
     // Input data
+
     input            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet.csv'
-    mature           = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/mature.fa'
-    hairpin          = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hairpin.fa'
-    mirna_gtf        = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hsa.gff3'
-    mirtrace_species = 'hsa'
-    skip_mirdeep     = true
+    fasta            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/genome.fa'
+
+    mirtrace_species         = 'hsa'
+    skip_mirdeep             = true
+    save_merged              = false
+    save_aligned_mirna_quant = false
+
 
+    filter_contamination = true
+    cdna                 = "https://huggingface.co/datasets/nf-core/smrnaseq/resolve/main/GRCh37/Homo_sapiens.GRCh37.cdna.all.fa"
+    ncrna                = "https://huggingface.co/datasets/nf-core/smrnaseq/resolve/main/GRCh37/Homo_sapiens.GRCh37.ncrna.fa"
+    trna                 = "https://huggingface.co/datasets/nf-core/smrnaseq/resolve/main/GRCh37/hg19-tRNAs.fa"
 }
 
 // Include illumina config to run test without additional profiles

diff --git a/conf/test_contamination_tech_reps.config b/conf/test_contamination_tech_reps.config
@@ -0,0 +1,42 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/smrnaseq -profile test_contamination_tech_reps,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+// Test covers techincal_repeats, skip_fastqc, filter_contamination and running without genome.
+
+params {
+    config_profile_name        = 'Test technical repeats profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus   = 2
+    max_memory = '6.GB'
+    max_time   = '6.h'
+
+    // Input data
+    input      = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet_technical_repeats_short.csv'
+
+    mirtrace_species         = 'hsa'
+    save_merged              = false
+    save_aligned_mirna_quant = false
+
+    skip_multiqc             = true
+    skip_mirdeep             = true
+    skip_fastqc              = true
+
+    filter_contamination = true
+    cdna                 = "https://huggingface.co/datasets/nf-core/smrnaseq/resolve/main/GRCh37/Homo_sapiens.GRCh37.cdna.all.fa"
+    ncrna                = "https://huggingface.co/datasets/nf-core/smrnaseq/resolve/main/GRCh37/Homo_sapiens.GRCh37.ncrna.fa"
+    trna                 = "https://huggingface.co/datasets/nf-core/smrnaseq/resolve/main/GRCh37/hg19-tRNAs.fa"
+}
+
+// Include illumina config to run test without additional profiles
+
+includeConfig 'protocol_illumina.config'
diff --git a/conf/test_mirgenedb.config b/conf/test_mirgenedb.config
@@ -20,21 +20,20 @@ params {
     max_time   = '6.h'
 
     // Input data
-    input            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet.csv'
+    input            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet_test_short.csv'
     fasta            = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/genome.fa'
 
     mirgenedb                = true
 
-    mirgenedb_mature         = "https://mirgenedb.org/fasta/hsa?mat=1"
-    mirgenedb_hairpin        = "https://mirgenedb.org/static/data/hsa/hsa-hg38-pri-30-30.fas"
-    mirgenedb_gff            = "https://mirgenedb.org/gff/hsa?sort=pos&all=1"
+    mirgenedb_mature         = "https://github.com/nf-core/test-datasets/raw/smrnaseq/MirGeneDB/mirgenedb_hsa_mature.fa"
+    mirgenedb_hairpin        = "https://github.com/nf-core/test-datasets/raw/smrnaseq/MirGeneDB/mirgenedb_hsa_hairpin.fa"
+    mirgenedb_gff            = "https://github.com/nf-core/test-datasets/raw/smrnaseq/MirGeneDB/mirgenedb_hsa.gff"
     mirgenedb_species        = "Hsa"
 
     skip_mirdeep             = true
     save_merged              = false
     save_aligned_mirna_quant = false
 
-    cleanup                  = true //Otherwise tests dont run through properly.
 }
 
 // Include illumina config to run test without additional profiles

diff --git a/conf/test_no_genome_nextflex.config → conf/test_nextflex.config b/conf/test_no_genome_nextflex.config → conf/test_nextflex.config
@@ -5,13 +5,14 @@
     Defines input files and everything required to run a fast and simple pipeline test.
 
     Use as follows:
-        nextflow run nf-core/smrnaseq -profile test,<docker/singularity>
+        nextflow run nf-core/smrnaseq -profile test_nextflex,<docker/singularity>
 
 ----------------------------------------------------------------------------------------
 */
+// This test profile tests nextflex without genome
 
 params {
-    config_profile_name        = 'Test profile'
+    config_profile_name        = 'Nextflex Test profile'
     config_profile_description = 'Minimal test dataset to check pipeline function'
 
     // Limit resources so that this can run on GitHub Actions
@@ -20,12 +21,14 @@ params {
     max_time   = '6.h'
 
     // Input data
-    input               = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet_test_nextflex.csv'
+    input               = 'https://raw.githubusercontent.com/nf-core/test-datasets/smrnaseq/samplesheet/v2.0/samplesheet_test_nextflex.csv'
     mature              = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/mature.fa'
     hairpin             = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hairpin.fa'
     mirna_gtf           = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hsa.gff3'
     mirtrace_species    = 'hsa'
 
+    skip_mirdeep             = true
+
 }
 
 // Include nextflex config to run test without additional profiles