Merge pull request #61 from nf-core/dev

Dev > Master, v1.0 release

.travis.yml

@@ -1,28 +1,33 @@
 sudo: required
-language: java
+language: python
 jdk: openjdk8
 services: docker
+python: '3.6'
+cache: pip
+matrix:
+  fast_finish: true
 
-before_install: docker pull nfcore/rnaseq:latest
+before_install:
+  # PRs to master are only ok if coming from dev branch
+  - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])'
+  # Pull the docker image first so the test doesn't wait for this
+  - docker pull nfcore/rnaseq
+  # Fake the tag locally so that the pipeline runs properly
+  - docker tag nfcore/rnaseq nfcore/rnaseq:1.0
 
 install:
   # Install Nextflow
-  - mkdir /tmp/nextflow
-  - cd /tmp/nextflow
+  - mkdir /tmp/nextflow && cd /tmp/nextflow
   - wget -qO- get.nextflow.io | bash
   - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow
   # Install nf-core/tools
-  - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools
-  - cd /tmp/nf-core-tools
-  - pip install --user -e .
-  # Make test directories
-  - mkdir ${TRAVIS_BUILD_DIR}/tests
-  - cd ${TRAVIS_BUILD_DIR}/tests
-
+  - pip install nf-core
+  # Reset
+  - mkdir ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests
 
 env:
-  - NXF_VER=0.30.1
-  - ''
+  - NXF_VER='0.31.1' # Specify a minimum NF version that should be tested and work
+  - NXF_VER='' # Plus: get the latest NF version and check that it works
 
 script:
   # Lint the pipeline code

CHANGELOG.md

@@ -1,5 +1,42 @@
-## nf-core/rnaseq v1.0dev
-This release marks the point where the pipeline was moved from SciLifeLab/NGI-RNAseq
-over to the new nf-core community, at nf-core/rnaseq.
+# nf-core/rnaseq
+
+## [Version 1.0](https://github.com/nf-core/rnaseq/releases/tag/1.0) - 2018-08-20
+
+This release marks the point where the pipeline was moved from [SciLifeLab/NGI-RNAseq](https://github.com/SciLifeLab/NGI-RNAseq)
+over to the new [nf-core](http://nf-co.re/) community, at [nf-core/rnaseq](https://github.com/nf-core/rnaseq).
 
 View the previous changelog at [SciLifeLab/NGI-RNAseq/CHANGELOG.md](https://github.com/SciLifeLab/NGI-RNAseq/blob/master/CHANGELOG.md)
+
+In addition to porting to the new nf-core community, the pipeline has had a number of major changes in this version.
+There have been 157 commits by 16 different contributors covering 70 different files in the pipeline: 7,357 additions and 8,236 deletions!
+
+In summary, the main changes are:
+
+* Rebranding and renaming throughout the pipeline to nf-core
+* Updating many parts of the pipeline config and style to meet nf-core standards
+* Support for GFF files in addition to GTF files
+    * Just use `--gff` instead of `--gtf` when specifying a file path
+* New command line options to skip various quality control steps
+* More safety checks when launching a pipeline
+    * Several new sanity checks - for example, that the specified reference genome exists
+* Improved performance with memory usage (especially STAR and Picard)
+* New BigWig file outputs for plotting coverage across the genome
+* Refactored gene body coverage calculation, now much faster and using much less memory
+* Bugfixes in the MultiQC process to avoid edge cases where it wouldn't run
+* MultiQC report now automatically attached to the email sent when the pipeline completes
+* New testing method, with data on GitHub
+    * Now run pipeline with `-profile test` instead of using bash scripts
+* Rewritten continuous integration tests with Travis CI
+* New explicit support for Singularity containers
+* Improved MultiQC support for DupRadar and featureCounts
+    * Now works for all users instead of just NGI Stockholm
+* New configuration for use on AWS batch
+* Updated config syntax to support latest versions of Nextflow
+* Built-in support for a number of new local HPC systems
+    * CCGA, GIS, UCT HEX, updates to UPPMAX, CFC, BINAC, Hebbe, c3se
+* Slightly improved documentation (more updates to come)
+* Updated software packages
+
+...and many more minor tweaks.
+
+Thanks to everyone who has worked on this release!

Dockerfile

@@ -4,4 +4,5 @@ LABEL authors="phil.ewels@scilifelab.se" \
     description="Docker image containing all requirements for the nfcore/rnaseq pipeline"
 
 COPY environment.yml /
-RUN conda env update -n root -f /environment.yml && conda clean -a
+RUN conda env create -f /environment.yml && conda clean -a
+ENV PATH /opt/conda/envs/nf-core-rnaseq-1.0/bin:$PATH

README.md

@@ -1,7 +1,7 @@
 # ![nfcore/rnaseq](docs/images/nfcore-rnaseq_logo.png)
 
 [![Build Status](https://travis-ci.org/nf-core/rnaseq.svg?branch=master)](https://travis-ci.org/nf-core/rnaseq)
-[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.30.1-brightgreen.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.31.1-brightgreen.svg)](https://www.nextflow.io/)
 [![Gitter](https://img.shields.io/badge/gitter-%20join%20chat%20%E2%86%92-4fb99a.svg)](https://gitter.im/nf-core/Lobby)
 
 [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/)

Singularity

@@ -3,12 +3,16 @@ Bootstrap:docker
 
 %labels
     MAINTAINER Phil Ewels <phil.ewels@scilifelab.se>
-    DESCRIPTION Container image containing all requirements for the nf-core/rnaseq pipeline
-    VERSION 1.0dev
+    DESCRIPTION Singularity image containing all requirements for the nf-core/rnaseq pipeline
+    VERSION 1.0
+
+%environment
+    PATH=/opt/conda/envs/nf-core-rnaseq-1.0/bin:$PATH
+    export PATH
 
 %files
     environment.yml /
 
 %post
-    /opt/conda/bin/conda env update -n root -f /environment.yml
+    /opt/conda/bin/conda env create -f /environment.yml
     /opt/conda/bin/conda clean -a

bin/scrape_software_versions.py

@@ -14,6 +14,7 @@
     'Picard MarkDuplicates': ['v_markduplicates.txt', r"([\d\.]+)-SNAPSHOT"],
     'Samtools': ['v_samtools.txt', r"samtools (\S+)"],
     'featureCounts': ['v_featurecounts.txt', r"featureCounts v(\S+)"],
+    'deepTools': ['v_deeptools.txt', r"bamCoverage (\S+)"],
     'StringTie': ['v_stringtie.txt', r"(\S+)"],
     'Preseq': ['v_preseq.txt', r"Version: (\S+)"],
     'RSeQC': ['v_rseqc.txt', r"read_duplication.py ([\d\.]+)"],
@@ -32,6 +33,7 @@
 results['featureCounts'] = '<span style="color:#999999;\">N/A</span>'
 results['StringTie'] = '<span style="color:#999999;\">N/A</span>'
 results['Preseq'] = '<span style="color:#999999;\">N/A</span>'
+results['deepTools'] = '<span style="color:#999999;\">N/A</span>'
 results['RSeQC'] = '<span style="color:#999999;\">N/A</span>'
 results['MultiQC'] = '<span style="color:#999999;\">N/A</span>'
 

conf/base.config

@@ -13,78 +13,76 @@ process {
 
   container = params.container
 
-  cpus = { check_max( 1 * task.attempt, 'cpus' ) }
+  cpus = { check_max( 2, 'cpus' ) }
   memory = { check_max( 8.GB * task.attempt, 'memory' ) }
   time = { check_max( 2.h * task.attempt, 'time' ) }
 
-  errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'terminate' }
-  maxRetries = 1
+  errorStrategy = { task.exitStatus in [1,143,137,104,134,139] ? 'retry' : 'terminate' }
+  maxRetries = 3
   maxErrors = '-1'
 
   // Process-specific resource requirements
   withName:makeSTARindex {
-    cpus = { check_max( 10 * task.attempt, 'cpus' ) }
+    cpus = { check_max( 10, 'cpus' ) }
     memory = { check_max( 80.GB * task.attempt, 'memory' ) }
     time = { check_max( 5.h * task.attempt, 'time' ) }
   }
   withName:makeHISATindex {
-    cpus = { check_max( 10 * task.attempt, 'cpus' ) }
+    cpus = { check_max( 10, 'cpus' ) }
     memory = { check_max( 80.GB * task.attempt, 'memory' ) }
     time = { check_max( 5.h * task.attempt, 'time' ) }
   }
-  withName:fastqc {
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
-  }
   withName:trim_galore {
-    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
     memory = { check_max( 16.GB * task.attempt, 'memory' ) }
     time = { check_max( 8.h * task.attempt, 'time' ) }
   }
   withName:star {
-    cpus = { check_max( 10 * task.attempt, 'cpus' ) }
+    cpus = { check_max (10, 'cpus')}
     memory = { check_max( 80.GB * task.attempt, 'memory' ) }
     time = { check_max( 8.h * task.attempt, 'time' ) }
   }
   withName:hisat2Align {
-    cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+    cpus = { check_max( 8, 'cpus' ) }
     memory = { check_max( 64.GB * task.attempt, 'memory' ) }
     time = { check_max( 8.h * task.attempt, 'time' ) }
   }
   withName:hisat2_sortOutput {
-    cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+    cpus = { check_max( 4, 'cpus' ) }
     memory = { check_max( 32.GB * task.attempt, 'memory' ) }
     time = { check_max( 8.h * task.attempt, 'time' ) }
   }
   withName:rseqc {
+    cpus = { check_max( 8, 'cpus' ) }
+    memory = { check_max( 32.GB * task.attempt, 'memory' ) }
+    time = { check_max( 7.h * task.attempt, 'time' ) }
+    errorStrategy = 'ignore'
+  }
+  withName:createBigWig {
     cpus = { check_max( 8 * task.attempt, 'cpus' ) }
     memory = { check_max( 32.GB * task.attempt, 'memory' ) }
     time = { check_max( 7.h * task.attempt, 'time' ) }
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
   }
   withName:genebody_coverage {
-    cpus = { check_max( 1 * task.attempt, 'cpus' ) }
+    cpus = { check_max( 1, 'cpus' ) }
     memory = { check_max( 32.GB * task.attempt, 'memory' ) }
     time = { check_max( 7.h * task.attempt, 'time' ) }
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
-  }
-  withName:preseq {
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
   }
   withName:markDuplicates {
-    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
     memory = { check_max( 16.GB * task.attempt, 'memory' ) }
   }
   withName:dupradar {
-    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+    cpus = { check_max( 1, 'cpus' ) }
+    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+  }
+  withName:featureCounts {
     memory = { check_max( 16.GB * task.attempt, 'memory' ) }
   }
   withName:sample_correlation {
-    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
     memory = { check_max( 16.GB * task.attempt, 'memory' ) }
   }
   withName:multiqc {
     memory = { check_max( 2.GB * task.attempt, 'memory' ) }
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
+    errorStrategy = 'ignore'
   }
   withName:get_software_versions {
     memory = { check_max( 2.GB, 'memory' ) }
@@ -94,7 +92,6 @@ process {
   withName:workflow_summary_mqc {
     memory = { check_max( 2.GB, 'memory' ) }
     cache = false
-    executor = 'local'
     errorStrategy = 'ignore'
   }
 }
@@ -105,5 +102,4 @@ params {
   max_cpus = 16
   max_time = 240.h
   igenomes_base = 's3://ngi-igenomes/igenomes/'
-  maxMultiqcEmailFileSize = 25.MB
 }

environment.yml

@@ -1,17 +1,17 @@
 # You can use this file to create a conda environment for this pipeline:
 #   conda env create -f environment.yml
-name: nfcore-rnaseq-1.0dev
+name: nf-core-rnaseq-1.0
 channels:
   - bioconda
   - conda-forge
   - defaults
 dependencies:
   - conda-forge::openjdk=8.0.144 # Needed for FastQC - conda build hangs without this
   - fastqc=0.11.7
-  - trim-galore=0.5
-  - star=2.6.0c
+  - trim-galore=0.5.0
+  - star=2.6.1a
   - hisat2=2.1.0
-  - picard=2.18.7
+  - picard=2.18.11
   - bioconductor-dupradar=1.8.0
   - conda-forge::r-data.table=1.11.4
   - conda-forge::r-gplots=3.0.1
@@ -21,7 +21,7 @@ dependencies:
   - rseqc=2.6.4
   - samtools=1.9
   - stringtie=1.3.4
-  - subread=1.6.1
+  - subread=1.6.2
   - gffread=0.9.9
+  - deeptools=3.1.1
   - multiqc=1.6
-  - deeptools=3.1.1