Adding support for fastq.gz.spring-files as input

assets/schema_input.json

@@ -47,17 +47,20 @@
                 "pattern": "^\\S+$",
                 "unique": ["patient", "sample"],
                 "anyOf": [
+                    {
+                        "dependentRequired": ["bam"]
+                    },
                     {
                         "dependentRequired": ["fastq_1"]
                     },
                     {
-                        "dependentRequired": ["bam"]
+                        "dependentRequired": ["spring_1"]
                     }
                 ],
                 "meta": ["lane"]
             },
             "fastq_1": {
-                "errorMessage": "FastQ file for reads 1 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'",
+                "errorMessage": "Gzipped FastQ file for reads 1 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'",
                 "anyOf": [
                     {
                         "type": "string",
@@ -72,7 +75,7 @@
                 "exists": true
             },
             "fastq_2": {
-                "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'",
+                "errorMessage": "Gzipped FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'",
                 "dependentRequired": ["fastq_1"],
                 "anyOf": [
                     {
@@ -87,6 +90,37 @@
                 "format": "file-path",
                 "exists": true
             },
+            "spring_1": {
+                "errorMessage": "Gzipped and spring-compressed FastQ file for reads 1 cannot contain spaces and must have extension '.fq.gz.spring' or '.fastq.gz.spring'",
+                "anyOf": [
+                    {
+                        "type": "string",
+                        "pattern": "^\\S+\\.f(ast)?q\\.gz.spring$"
+                    },
+                    {
+                        "type": "string",
+                        "maxLength": 0
+                    }
+                ],
+                "format": "file-path",
+                "exists": true
+            },
+            "spring_2": {
+                "errorMessage": "Gzipped and spring-compressed FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz.spring' or '.fastq.gz.spring'",
+                "dependentRequired": ["spring_1"],
+                "anyOf": [
+                    {
+                        "type": "string",
+                        "pattern": "^\\S+\\.f(ast)?q\\.gz.spring$"
+                    },
+                    {
+                        "type": "string",
+                        "maxLength": 0
+                    }
+                ],
+                "format": "file-path",
+                "exists": true
+            },
             "table": {
                 "errorMessage": "Recalibration table cannot contain spaces and must have extension '.table'",
                 "anyOf": [

conf/modules/modules.config

@@ -59,6 +59,10 @@ process {
         ]
     }
 
+    withName: 'NFCORE_SAREK:SAREK:SPRING_DECOMPRESS_.*' {
+        ext.prefix = { "${spring.simpleName}" }
+    }
+
     withName: 'MOSDEPTH' {
         ext.args   = { !params.wes ? "-n --fast-mode --by 500" : ""}
         ext.prefix = {

conf/test/alignment_from_fastq_and_spring.config

@@ -0,0 +1,15 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/sarek -profile test,<extra_test_profile>,<docker/singularity> --outdir <OUTDIR>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+*/
+
+params {
+    input = "${projectDir}/tests/csv/3.0/fastq_and_spring.csv"
+    tools = null
+}

modules.json

@@ -449,6 +449,11 @@
                         "git_sha": "2f3db6f45147ebbb56b371536e31bdf622b5bfee",
                         "installed_by": ["modules", "vcf_annotate_snpeff"]
                     },
+                    "spring/decompress": {
+                        "branch": "master",
+                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "installed_by": ["modules"]
+                    },
                     "strelka/germline": {
                         "branch": "master",
                         "git_sha": "e8f2c77a6e4174ee0a48d073d4cc8ff06c44bb4c",

nextflow.config

@@ -296,6 +296,8 @@ profiles {
     }
 
     // Extra test profiles for more complete CI
+    // TO-DO: Indentation!
+    alignment_from_fastq_and_spring { includeConfig 'conf/test/alignment_from_fastq_and_spring.config' }
     alignment_to_fastq         { includeConfig 'conf/test/alignment_to_fastq.config' }
     annotation                 { includeConfig 'conf/test/annotation.config' }
     markduplicates_bam         { includeConfig 'conf/test/markduplicates_bam.config' }

subworkflows/local/samplesheet_to_channel/main.nf

@@ -33,32 +33,43 @@ workflow  SAMPLESHEET_TO_CHANNEL{
 
     main:
     ch_from_samplesheet.dump(tag:"ch_from_samplesheet")
-    input_sample = ch_from_samplesheet.map{ meta, fastq_1, fastq_2, table, cram, crai, bam, bai, vcf, variantcaller ->
+    input_sample = ch_from_samplesheet.map{ meta, fastq_1, fastq_2, spring_1, spring_2, table, cram, crai, bam, bai, vcf, variantcaller ->
         // generate patient_sample key to group lanes together
-        [ meta.patient + meta.sample, [meta, fastq_1, fastq_2, table, cram, crai, bam, bai, vcf, variantcaller] ]
+        [ meta.patient + meta.sample, [meta, fastq_1, fastq_2, spring_1, spring_2, table, cram, crai, bam, bai, vcf, variantcaller] ]
     }.tap{ ch_with_patient_sample } // save the channel
     .groupTuple() //group by patient_sample to get all lanes
     .map { patient_sample, ch_items ->
         // get number of lanes per sample
         [ patient_sample, ch_items.size() ]
     }.combine(ch_with_patient_sample, by: 0) // for each entry add numLanes
     .map { patient_sample, num_lanes, ch_items ->
-        (meta, fastq_1, fastq_2, table, cram, crai, bam, bai, vcf, variantcaller) = ch_items
+        (meta, fastq_1, fastq_2, spring_1, spring_2, table, cram, crai, bam, bai, vcf, variantcaller) = ch_items
         if (meta.lane && fastq_2) {
-            meta           = meta + [id: "${meta.sample}-${meta.lane}".toString()]
-            def CN         = seq_center ? "CN:${seq_center}\\t" : ''
-
-            def flowcell   = flowcellLaneFromFastq(fastq_1)
-            // Don't use a random element for ID, it breaks resuming
-            def read_group = "\"@RG\\tID:${flowcell}.${meta.sample}.${meta.lane}\\t${CN}PU:${meta.lane}\\tSM:${meta.patient}_${meta.sample}\\tLB:${meta.sample}\\tDS:${fasta}\\tPL:${seq_platform}\""
-
-            meta           = meta - meta.subMap('lane') + [num_lanes: num_lanes.toInteger(), read_group: read_group.toString(), data_type: 'fastq', size: 1]
+            meta = meta + [id: "${meta.sample}-${meta.lane}".toString(), data_type: "fastq_gz", num_lanes: num_lanes.toInteger(), size: 1]
 
             if (step == 'mapping') return [ meta, [ fastq_1, fastq_2 ] ]
             else {
                 error("Samplesheet contains fastq files but step is `$step`. Please check your samplesheet or adjust the step parameter.\nhttps://nf-co.re/sarek/usage#input-samplesheet-configurations")
             }
 
+        // start from TWO spring-files - one with R1 and one with R2
+        } else if (meta.lane && spring_1 && spring_2) {
+            meta = meta + [id: "${meta.sample}-${meta.lane}".toString(), data_type: "two_fastq_gz_spring", num_lanes: num_lanes.toInteger(), size: 1]
+
+            if (step == 'mapping') return [ meta, [ spring_1, spring_2 ] ]
+            else {
+                error("Samplesheet contains spring files (in columns `spring_1` and `spring_2`) but step is `$step`. Please check your samplesheet or adjust the step parameter.\nhttps://nf-co.re/sarek/usage#input-samplesheet-configurations")
+            }
+
+        // start from ONE spring-file containing both R1 and R2
+        } else if (meta.lane && spring_1 && !spring_2) {
+            meta = meta + [id: "${meta.sample}-${meta.lane}".toString(), data_type: "one_fastq_gz_spring", num_lanes: num_lanes.toInteger(), size: 1]
+
+            if (step == 'mapping') return [ meta, [ spring_1 ] ]
+            else {
+                error("Samplesheet contains a spring file (in columns `spring_1`) but step is `$step`. Please check your samplesheet or adjust the step parameter.\nhttps://nf-co.re/sarek/usage#input-samplesheet-configurations")
+            }
+
         // start from BAM
         } else if (meta.lane && bam) {
             if (step != 'mapping' && !bai) {
@@ -270,36 +281,3 @@ Joint germline variant calling also requires intervals in order to genotype the
     emit:
     input_sample
     }
-
-/*
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    FUNCTIONS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-*/
-// Parse first line of a FASTQ file, return the flowcell id and lane number.
-def flowcellLaneFromFastq(path) {
-    // expected format:
-    // xx:yy:FLOWCELLID:LANE:... (seven fields)
-    // or
-    // FLOWCELLID:LANE:xx:... (five fields)
-    def line
-    path.withInputStream {
-        InputStream gzipStream = new java.util.zip.GZIPInputStream(it)
-        Reader decoder = new InputStreamReader(gzipStream, 'ASCII')
-        BufferedReader buffered = new BufferedReader(decoder)
-        line = buffered.readLine()
-    }
-    assert line.startsWith('@')
-    line = line.substring(1)
-    def fields = line.split(':')
-    String fcid
-
-    if (fields.size() >= 7) {
-        // CASAVA 1.8+ format, from  https://support.illumina.com/help/BaseSpace_OLH_009008/Content/Source/Informatics/BS/FileFormat_FASTQ-files_swBS.htm
-        // "@<instrument>:<run number>:<flowcell ID>:<lane>:<tile>:<x-pos>:<y-pos>:<UMI> <read>:<is filtered>:<control number>:<index>"
-        fcid = fields[2]
-    } else if (fields.size() == 5) {
-        fcid = fields[0]
-    }
-    return fcid
-}

tests/csv/3.0/fastq_and_spring.csv

@@ -0,0 +1,4 @@
+patient,sex,status,sample,lane,fastq_1,fastq_2,spring_1,spring_2
+test,XX,0,test,test_L1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz,,
+test2,XX,0,test2,test2_L1,,,https://raw.githubusercontent.com/nf-core/test-datasets/sarek3/data/genomics/homo_sapiens/illumina/spring/test_1.fastq.gz.spring,https://raw.githubusercontent.com/nf-core/test-datasets/sarek3/data/genomics/homo_sapiens/illumina/spring/test_2.fastq.gz.spring
+test3,XX,0,test3,test3_L1,,,https://raw.githubusercontent.com/nf-core/test-datasets/sarek3/data/genomics/homo_sapiens/illumina/spring/test_R1_R2.fastq.gz.spring,