From 338c62a8cd756cdeea6f6dccb2cfd676ecdf85e1 Mon Sep 17 00:00:00 2001 From: alneberg Date: Mon, 28 Jan 2019 14:05:21 +0100 Subject: [PATCH 1/9] Stashing awsbatch testing --- conf/aws-batch.config | 4 ++++ conf/docker.config | 2 +- conf/resources.config | 1 - main.nf | 1 - 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conf/aws-batch.config b/conf/aws-batch.config index 33850cd686..cbd7da21ed 100644 --- a/conf/aws-batch.config +++ b/conf/aws-batch.config @@ -33,6 +33,10 @@ process { maxRetries = 4 cpus = 2 memory = 8.GB + time = { check_max( 18.h * task.attempt, 'time' ) } + + scratch = true + beforeScript = "sudo yum -y install wget mdadm; wget https://raw.githubusercontent.com/alneberg/bu/master/scripts/aws/raid0_xvd.sh; sudo bash raid0_xvd.sh" withName:RunBcftoolsStats { cpus = 1 diff --git a/conf/docker.config b/conf/docker.config index 1dd69206ff..299c47ed84 100644 --- a/conf/docker.config +++ b/conf/docker.config @@ -10,5 +10,5 @@ docker { enabled = true fixOwnership = true - runOptions = "-u \$(id -u):\$(id -g)" + runOptions = "-u \$(id -u):\$(id -g) -v /efs:/tmp" } diff --git a/conf/resources.config b/conf/resources.config index 1a7dc84a77..d1e605c3b5 100644 --- a/conf/resources.config +++ b/conf/resources.config @@ -78,7 +78,6 @@ process { cpus = { check_max( 1, 'cpus' ) } // Increase memory quadratically memory = { check_max( 7.GB * 2 * task.attempt, 'memory' ) } - time = { check_max( 5.h * task.attempt, 'time' ) } } withName:RunGenotypeGVCFs { cpus = { check_max( 1, 'cpus' ) } diff --git a/main.nf b/main.nf index 23b6e246ac..4e81261cf1 100644 --- a/main.nf +++ b/main.nf @@ -352,7 +352,6 @@ process CreateRecalibrationTable { BaseRecalibrator \ --input ${bam} \ --output ${idSample}.recal.table \ - --TMP_DIR /tmp \ -R ${genomeFile} \ -L ${intervals} \ --known-sites ${dbsnp} \ From 1bbf9d76e294b9e6e2e1574a0d2d2dddbb4e5353 Mon Sep 17 00:00:00 2001 From: alneberg Date: Mon, 28 Jan 2019 14:22:45 +0100 Subject: [PATCH 2/9] A new test data tsv file for dream data --- Sarek-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sarek-data b/Sarek-data index 03b5a18b2b..c523d6d1d7 160000 --- a/Sarek-data +++ b/Sarek-data @@ -1 +1 @@ -Subproject commit 03b5a18b2bdba3dac6307e27a5b5c7e5fec3bd54 +Subproject commit c523d6d1d76ea40c3c155f2363268bec698a56a1 From 22a92ef090854197c3f5f5c097414235efa5ffd0 Mon Sep 17 00:00:00 2001 From: alneberg Date: Tue, 5 Feb 2019 17:25:06 +0100 Subject: [PATCH 3/9] Remove beforeScript, not needed when the AMI was fixed --- conf/aws-batch.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/aws-batch.config b/conf/aws-batch.config index cbd7da21ed..777271e71c 100644 --- a/conf/aws-batch.config +++ b/conf/aws-batch.config @@ -36,7 +36,6 @@ process { time = { check_max( 18.h * task.attempt, 'time' ) } scratch = true - beforeScript = "sudo yum -y install wget mdadm; wget https://raw.githubusercontent.com/alneberg/bu/master/scripts/aws/raid0_xvd.sh; sudo bash raid0_xvd.sh" withName:RunBcftoolsStats { cpus = 1 From 710c0488a651d43ddde824870dbd552eae1707a1 Mon Sep 17 00:00:00 2001 From: alneberg Date: Tue, 5 Feb 2019 17:25:52 +0100 Subject: [PATCH 4/9] No efs tmp mapping in runoptions --- conf/docker.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker.config b/conf/docker.config index 299c47ed84..1dd69206ff 100644 --- a/conf/docker.config +++ b/conf/docker.config @@ -10,5 +10,5 @@ docker { enabled = true fixOwnership = true - runOptions = "-u \$(id -u):\$(id -g) -v /efs:/tmp" + runOptions = "-u \$(id -u):\$(id -g)" } From 88b07e19483ec6d5be3a33c7fa6de6d3f6a499f6 Mon Sep 17 00:00:00 2001 From: alneberg Date: Tue, 5 Feb 2019 17:37:21 +0100 Subject: [PATCH 5/9] Merged sarek-data repo --- Sarek-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sarek-data b/Sarek-data index c523d6d1d7..10d3f8fd99 160000 --- a/Sarek-data +++ b/Sarek-data @@ -1 +1 @@ -Subproject commit c523d6d1d76ea40c3c155f2363268bec698a56a1 +Subproject commit 10d3f8fd99720f0585e204067094dc1ec3a28721 From 152ec5c9eb16690f70e04b36b12a04141c468278 Mon Sep 17 00:00:00 2001 From: alneberg Date: Tue, 5 Feb 2019 17:40:54 +0100 Subject: [PATCH 6/9] Added a normal-only dream testdata tsv --- Sarek-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sarek-data b/Sarek-data index 10d3f8fd99..68d43ed200 160000 --- a/Sarek-data +++ b/Sarek-data @@ -1 +1 @@ -Subproject commit 10d3f8fd99720f0585e204067094dc1ec3a28721 +Subproject commit 68d43ed200ff7a1a8db88864349a98ef2d0669cb From 4ebfa43484d17d7c2ffef7b892b5c9c91f4f8ae4 Mon Sep 17 00:00:00 2001 From: alneberg Date: Mon, 11 Feb 2019 14:31:55 +0100 Subject: [PATCH 7/9] Add back time restriction, deal with that later --- conf/resources.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/resources.config b/conf/resources.config index d1e605c3b5..1a7dc84a77 100644 --- a/conf/resources.config +++ b/conf/resources.config @@ -78,6 +78,7 @@ process { cpus = { check_max( 1, 'cpus' ) } // Increase memory quadratically memory = { check_max( 7.GB * 2 * task.attempt, 'memory' ) } + time = { check_max( 5.h * task.attempt, 'time' ) } } withName:RunGenotypeGVCFs { cpus = { check_max( 1, 'cpus' ) } From 25606942df41a4d71de4a0299c828ed7fe5a4cce Mon Sep 17 00:00:00 2001 From: alneberg Date: Mon, 11 Feb 2019 14:58:56 +0100 Subject: [PATCH 8/9] Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b28b351c7..2d20f79e54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - [#719](https://github.com/SciLifeLab/Sarek/pull/719) - `vepCacheVersion` is now defined in `conf/genomes.config` or `conf/igenomes.config` - [#719](https://github.com/SciLifeLab/Sarek/pull/719) - `snpeff` and `vep` containers are now built with conda - [#716](https://github.com/SciLifeLab/Sarek/pull/716) - Update paths to containers and iGenomes +- [#724](https://github.com/SciLifeLab/Sarek/pull/724) - Improved AwsBatch configuration ### `Added` - [#719](https://github.com/SciLifeLab/Sarek/pull/719) - Possibility to use cache wen annotating with `snpEff` and `VEP` From 66900afdef861d351f0fa2210ba5ac023c56d6a2 Mon Sep 17 00:00:00 2001 From: alneberg Date: Mon, 11 Feb 2019 17:02:33 +0100 Subject: [PATCH 9/9] Updated Sarek-data to latest master --- Sarek-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sarek-data b/Sarek-data index 68d43ed200..456c73f3e5 160000 --- a/Sarek-data +++ b/Sarek-data @@ -1 +1 @@ -Subproject commit 68d43ed200ff7a1a8db88864349a98ef2d0669cb +Subproject commit 456c73f3e5888b5122c0f49190b04f10789704fe