Merge remote-tracking branch 'upstream/dsl2' into dsl2_dev_merge

.github/workflows/ci.yml

@@ -19,47 +19,30 @@ jobs:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        # Nextflow versions: check pipeline minimum and current latest
-        nxf_ver: ['20.04.0', '']
+        # Nextflow versions: check pipeline minimum
+        nxf_ver: ['20.11.0-edge']
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
 
-      - name: Check if Dockerfile or Conda environment changed
-        uses: technote-space/get-diff-action@v4
-        with:
-          FILES: |
-            Dockerfile
-            environment.yml
-      - name: Build new docker image
-        if: env.MATCHED_FILES
-        run: docker build --no-cache . -t nfcore/sarek:dev
-
-      - name: Pull docker image
-        if: ${{ !env.MATCHED_FILES }}
-        run: |
-          docker pull nfcore/sarek:dev
-          docker tag nfcore/sarek:dev nfcore/sarek:dev
-
       - name: Install Nextflow
         env:
           CAPSULE_LOG: none
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-
       - name: Run pipeline with test data
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test,docker
 
-  annotation:
+  aligner:
+    name: Run aligner tests
     env:
       NXF_ANSI_LOG: false
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        tools: [snpeff, vep]
-        species: [WBcel235]
+        aligner: [bwa-mem, bwa-mem2]
     steps:
       - uses: actions/checkout@v2
       - name: Install Nextflow
@@ -68,15 +51,12 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
-      - name: Pull docker image
-        run: |
-          docker pull nfcore/sarek:dev
-          docker pull nfcore/sarek${{ matrix.tools }}:dev.${{ matrix.species }}
-      - name: Run annotation test
-        run: nextflow run ${GITHUB_WORKSPACE} -profile test_annotation,docker --tools ${{ matrix.tools }}
+          NXF_VER: '20.11.0-edge'
+      - name: Run ${{ matrix.profile }} test
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner ${{ matrix.aligner }}
 
   germline:
+    name: Run input from a folder test and restart from step tests
     env:
       NXF_ANSI_LOG: false
     runs-on: ubuntu-latest
@@ -91,99 +71,99 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
-      - name: Pull docker image
-        run: docker pull nfcore/sarek:dev
+          NXF_VER: '20.11.0-edge'
       - name: Get test data
         run: git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data
-      - name: Run germline test
-        run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input data/testdata/tiny/normal --saved_bam_mapped
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step prepare_recalibration -resume
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step recalibrate -resume
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step variantCalling
+      - name: Run germline test with ${{ matrix.markduplicates }} --step mapping
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input data/testdata/tiny/normal --save_bam_mapped
+      - name: Run germline test with ${{ matrix.markduplicates }} --step prepare_recalibration
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step prepare_recalibration -resume
+      - name: Run germline test with ${{ matrix.markduplicates }} --step recalibrate
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step recalibrate -resume
+      - name: Run germline test with ${{ matrix.markduplicates }} --step variantCalling
+        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.markduplicates }} --input=false --step variantCalling
 
-  minimal:
-    env:
-      NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        genome: [smallerGRCh37, minimalGRCh37]
-        intervals: [--no_intervals, '']
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install Nextflow
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-        env:
-          # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
-      - name: Pull docker image
-        run: docker pull nfcore/sarek:dev
-      - name: Run test for minimal genomes
-        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --skipQC all --genome ${{ matrix.genome }} ${{ matrix.intervals }} --tools Manta,mpileup,Strelka,FreeBayes
+  # annotation:
+  #   env:
+  #     NXF_ANSI_LOG: false
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       tools: [snpeff, vep]
+  #       species: [WBcel235]
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Install Nextflow
+  #       run: |
+  #         wget -qO- get.nextflow.io | bash
+  #         sudo mv nextflow /usr/local/bin/
+  #       env:
+  #         # Only check Nextflow pipeline minimum version
+  #         NXF_VER: '20.11.0-edge'
+  #     - name: Pull docker image
+  #       run: |
+  #         docker pull nfcore/sarek${{ matrix.tools }}:dev.${{ matrix.species }}
+  #     - name: Run annotation test
+  #       run: nextflow run ${GITHUB_WORKSPACE} -profile test_annotation,docker --tools ${{ matrix.tools }}
 
-  profile:
-    env:
-      NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        profile: [test_split_fastq, test_targeted, test_trimming, test_use_gatk_spark, test_umi_tso, test_umi_qiaseq]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install Nextflow
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-        env:
-          # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
-      - name: Pull docker image
-        run: docker pull nfcore/sarek:dev
-      - name: Run ${{ matrix.profile }} test
-        run: nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker
+  # minimal:
+  #   env:
+  #     NXF_ANSI_LOG: false
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       genome: [smallerGRCh37, minimalGRCh37]
+  #       intervals: [--no_intervals, '']
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Install Nextflow
+  #       run: |
+  #         wget -qO- get.nextflow.io | bash
+  #         sudo mv nextflow /usr/local/bin/
+  #       env:
+  #         # Only check Nextflow pipeline minimum version
+  #         NXF_VER: '20.11.0-edge'
+  #     - name: Run test for minimal genomes
+  #       run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --skipQC all --genome ${{ matrix.genome }} ${{ matrix.intervals }} --tools Manta,mpileup,Strelka,FreeBayes
 
-  aligner:
-    env:
-      NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        aligner: [bwa-mem, bwa-mem2]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install Nextflow
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-        env:
-          # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
-      - name: Pull docker image
-        run: docker pull nfcore/sarek:dev
-      - name: Run ${{ matrix.profile }} test
-        run: nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner ${{ matrix.aligner }}
+  # profile:
+  #   env:
+  #     NXF_ANSI_LOG: false
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       profile: [test_split_fastq, test_targeted, test_trimming, test_no_gatk_spark, test_umi_tso, test_umi_qiaseq]
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Install Nextflow
+  #       run: |
+  #         wget -qO- get.nextflow.io | bash
+  #         sudo mv nextflow /usr/local/bin/
+  #       env:
+  #         # Only check Nextflow pipeline minimum version
+  #         NXF_VER: '20.11.0-edge'
+  #     - name: Run ${{ matrix.profile }} test
+  #       run: nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker
 
   tools:
+    name: Run tool tests
     env:
       NXF_ANSI_LOG: false
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        tool: [Haplotypecaller, Freebayes, Manta, mpileup, MSIsensor, Strelka, TIDDIT]
+        # tool: [Haplotypecaller, Freebayes, Manta, mpileup, MSIsensor, Strelka, TIDDIT]
+        tool: [Haplotypecaller, Strelka]
         intervals: [--no_intervals, '']
         exclude:
-          - tool: Manta
-            intervals: --no_intervals
-          - tool: MSIsensor
-            intervals: --no_intervals
+        #   - tool: Manta
+        #     intervals: --no_intervals
+        #   - tool: MSIsensor
+        #     intervals: --no_intervals
           - tool: Strelka
             intervals: --no_intervals
-          - tool: TIDDIT
-            intervals: --no_intervals
+        #   - tool: TIDDIT
+        #     intervals: --no_intervals
     steps:
       - uses: actions/checkout@v2
       - name: Install Nextflow
@@ -192,8 +172,6 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
-      - name: Pull docker image
-        run: docker pull nfcore/sarek:dev
+          NXF_VER: '20.11.0-edge'
       - name: Run ${{ matrix.tool }} test
         run: nextflow run ${GITHUB_WORKSPACE} -profile test_tool,docker --tools ${{ matrix.tool }} ${{ matrix.intervals }}

CHANGELOG.md

@@ -9,8 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- [#238](https://github.com/nf-core/sarek/pull/238) -Add subworkflow for building all the indices
+- [#241](https://github.com/nf-core/sarek/pull/241) -Add modules and workflows parts for preprocessing steps
+
 ### Changed
 
+- [#233](https://github.com/nf-core/sarek/pull/233) - Switch `bwa 0.7.17` for `bwa-mem2 2.0`
+- [#234](https://github.com/nf-core/sarek/pull/234) -Switching to DSL2
+- [#234](https://github.com/nf-core/sarek/pull/234) - Update Nextflow `19.10.0` -> `20.04.1`
 - [#333](https://github.com/nf-core/sarek/pull/333) - Bump `Sarek` version to `3.0dev`
 
 ### Fixed

README.md

@@ -2,7 +2,7 @@
 
 > **An open-source analysis pipeline to detect germline or somatic variants from whole genome or targeted sequencing**
 
-[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.04.0-brightgreen.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A520.11.0--edge-brightgreen.svg)](https://www.nextflow.io/)
 [![nf-core](https://img.shields.io/badge/nf--core-pipeline-brightgreen.svg)](https://nf-co.re/)
 [![DOI](https://zenodo.org/badge/184289291.svg)](https://zenodo.org/badge/latestdoi/184289291)
 

assets/multiqc_config.yaml

@@ -19,6 +19,7 @@ top_modules:
      name: 'FastQC'
      path_filters_exclude: 
         - '*trimmed_fastqc*'
+- 'cutadapt'
 - 'fastqc':
      name: 'FastQC after trimming'
      info: 'FastQC after applying TrimGalore.'

bin/scrape_software_versions.py

@@ -6,54 +6,60 @@
 regexes = {
     "ASCAT": ["v_ascat.txt", r"Version:       (\S+)"],
     "AlleleCount": ["v_allelecount.txt", r"(\S+)"],
+    "bcftools": ["v_bcftools.txt", r"bcftools (\S+)"],
     "BWA": ["v_bwa.txt", r"Version: (\S+)"],
+    "BWA-MEM2": ["v_bwamem2.txt", r"(\S+)"],
     "CNVkit": ["v_cnvkit.txt", r"(\S+)"],
     "Control-FREEC": ["v_controlfreec.txt", r"Control-FREEC\s(\S+)"],
     "FastQC": ["v_fastqc.txt", r"FastQC v(\S+)"],
     "FreeBayes": ["v_freebayes.txt", r"version:  v(\d\.\d\.\d+)"],
     "GATK": ["v_gatk.txt", r"Version:(\S+)"],
+    "htslib": ["v_samtools.txt", r"htslib (\S+)"],
     "Manta": ["v_manta.txt", r"([0-9.]+)"],
     "MultiQC": ["v_multiqc.txt", r"multiqc, version (\S+)"],
+    "msisensor": ["v_msisensor.txt", r"Version: v(\S+)"],
     "Nextflow": ["v_nextflow.txt", r"(\S+)"],
+    "nf-core/sarek": ["v_pipeline.txt", r"(\S+)"],
     "QualiMap": ["v_qualimap.txt", r"QualiMap v.(\S+)"],
     "R": ["v_r.txt", r"R version (\S+)"],
+    "samtools": ["v_samtools.txt", r"samtools (\S+)"],
     "SnpEff": ["v_snpeff.txt", r"SnpEff\s(\S+)"],
     "Strelka": ["v_strelka.txt", r"([0-9.]+)"],
     "TIDDIT": ["v_tiddit.txt", r"TIDDIT-(\S+)"], 
     "Trim Galore": ["v_trim_galore.txt", r"version (\S+)"], 
-    "VEP": ["v_vep.txt", r"ensembl-vep          : (\S+)"],
-    "bcftools": ["v_bcftools.txt", r"bcftools (\S+)"],
-    "htslib": ["v_samtools.txt", r"htslib (\S+)"],
-    "msisensor": ["v_msisensor.txt", r"Version: v(\S+)"],
-    "nf-core/sarek": ["v_pipeline.txt", r"(\S+)"],
-    "samtools": ["v_samtools.txt", r"samtools (\S+)"],
-    "vcftools": ["v_vcftools.txt", r"([0-9.]+)"]
+    "vcftools": ["v_vcftools.txt", r"([0-9.]+)"],
+    "VEP": ["v_vep.txt", r"ensembl-vep          : (\S+)"]
 }
 results = OrderedDict()
 results["nf-core/sarek"] = '<span style="color:#999999;">N/A</span>'
 results["Nextflow"] = '<span style="color:#999999;">N/A</span>'
+
 results["BWA"] = '<span style="color:#999999;">N/A</span>'
+results["BWA-MEM2"] = '<span style="color:#999999;">N/A</span>'
 results["GATK"] = '<span style="color:#999999;">N/A</span>'
-results["FreeBayes"] = '<span style="color:#999999;">N/A</span>'
+results["htslib"] = '<span style="color:#999999;">N/A</span>'
 results["samtools"] = '<span style="color:#999999;">N/A</span>'
-results["Strelka"] = '<span style="color:#999999;">N/A</span>'
-results["Manta"] = '<span style="color:#999999;">N/A</span>'
-results["TIDDIT"] = '<span style="color:#999999;">N/A</span>' 
-results["AlleleCount"] = '<span style="color:#999999;">N/A</span>'
+
 results["ASCAT"] = '<span style="color:#999999;">N/A</span>'
+results["AlleleCount"] = '<span style="color:#999999;">N/A</span>'
+results["CNVkit"] = '<span style="color:#999999;">N/A</span>'
 results["Control-FREEC"] = '<span style="color:#999999;">N/A</span>'
+results["FreeBayes"] = '<span style="color:#999999;">N/A</span>'
+results["Manta"] = '<span style="color:#999999;">N/A</span>'
 results["msisensor"] = '<span style="color:#999999;">N/A</span>'
+results["R"] = '<span style="color:#999999;">N/A</span>'
+results["Strelka"] = '<span style="color:#999999;">N/A</span>'
+results["TIDDIT"] = '<span style="color:#999999;">N/A</span>' 
+
 results["SnpEff"] = '<span style="color:#999999;">N/A</span>'
 results["VEP"] = '<span style="color:#999999;">N/A</span>'
-results["MultiQC"] = '<span style="color:#999999;">N/A</span>'
-results["FastQC"] = '<span style="color:#999999;">N/A</span>'
+
 results["bcftools"] = '<span style="color:#999999;">N/A</span>'
-results["CNVkit"] = '<span style="color:#999999;">N/A</span>'
-results["htslib"] = '<span style="color:#999999;">N/A</span>'
+results["FastQC"] = '<span style="color:#999999;">N/A</span>'
+results["MultiQC"] = '<span style="color:#999999;">N/A</span>'
 results["QualiMap"] = '<span style="color:#999999;">N/A</span>'
 results["Trim Galore"] = '<span style="color:#999999;">N/A</span>' 
 results["vcftools"] = '<span style="color:#999999;">N/A</span>'
-results["R"] = '<span style="color:#999999;">N/A</span>'
 
 # Search each file using its regex
 for k, v in regexes.items():