diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb6c2af74..893c6e6153 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,6 @@ on: push: branches: - '*' - - '!refs/tags/.*' tags-ignore: - '*' pull_request: @@ -14,7 +13,6 @@ on: jobs: build: name: Build Nextflow - if: "!contains(github.event.head_commit.message, '[ci skip]') && (github.event == 'push' || github.repository != github.event.pull_request.head.repo.full_name)" runs-on: ubuntu-latest timeout-minutes: 120 strategy: diff --git a/docs/awscloud.rst b/docs/awscloud.rst index abdd6fa829..8eaf0aec7c 100644 --- a/docs/awscloud.rst +++ b/docs/awscloud.rst @@ -32,7 +32,7 @@ See :ref:`AWS configuration` for more details. AWS IAM policies ================= -`AIM policies `_ are the mechanism used by AWS to +`IAM policies `_ are the mechanism used by AWS to defines permissions for IAM identities. In order to access certain AWS services, the proper policies must be attached to the identity associated to the AWS credentials. @@ -76,10 +76,11 @@ Minimal permissions policies to be attached to the AWS account used by Nextflow S3 policies ------------ -Nextflow requires policies also to access `S3 buckets `_ in order to:: -- use the workdir -- pull input data -- publish results +Nextflow requires policies also to access `S3 buckets `_ in order to: + +1. use the workdir +2. pull input data +3. publish results Depending on the pipeline configuration, the above actions can be done all in a single bucket but, more likely, spread across multiple buckets. Once the list of buckets used by the pipeline is identified, there are two alternative ways to give Nextflow access to these buckets: @@ -165,10 +166,10 @@ Get started ------------- 1 - In the AWS Console, create a `Compute environment `_ (CE) in your AWS Batch Service. - * if are using a custom AMI (see following sections), the AMI ID must be specified in the CE configuration - * make sure to select an AMI (either custom or existing) with Docker installed (see following sections) - * make sure the policy ``AmazonS3FullAccess`` (granting access to S3 buckets) is attached to the instance role configured for the CE - * if you plan to use Docker images from Amazon ECS container, make sure the ``AmazonEC2ContainerServiceforEC2Role`` policy is also attached to the instance role + 1.1 - if are using a custom AMI (see following sections), the AMI ID must be specified in the CE configuration + 1.2 - make sure to select an AMI (either custom or existing) with Docker installed (see following sections) + 1.3 - make sure the policy ``AmazonS3FullAccess`` (granting access to S3 buckets) is attached to the instance role configured for the CE + 1.4 - if you plan to use Docker images from Amazon ECS container, make sure the ``AmazonEC2ContainerServiceforEC2Role`` policy is also attached to the instance role 2 - In the AWS Console, create (at least) one `Job Queue `_ and bind it to the Compute environment @@ -186,9 +187,10 @@ Configuration When configuring your pipeline: -- import the `nf-amazon` plugin -- specify the AWS Batch :ref:`executor` -- specify one or more AWS Batch queues for the execution by using the :ref:`process-queue` directive. +1 - import the `nf-amazon` plugin +2 - specify the AWS Batch :ref:`executor` +3 - specify one or more AWS Batch queues for the execution by using the :ref:`process-queue` directive +4 - specify the AWS job container properties by using the :ref:`process-containerOptions` directive. An example ``nextflow.config`` file is shown below:: @@ -200,6 +202,7 @@ An example ``nextflow.config`` file is shown below:: executor = 'awsbatch' queue = 'my-batch-queue' container = 'quay.io/biocontainers/salmon' + containerOptions = '--shm-size 16000000 --ulimit nofile=1280:2560 --ulimit nproc=16:32' } aws { @@ -212,19 +215,62 @@ An example ``nextflow.config`` file is shown below:: Different queues bound to the same or different Compute environments can be configured according to each process' requirements. +Container Options +================= + +As of version ``21.12.0-edge``, the use of the Nextflow :ref:`process-containerOptions` directive is supported to fine control +the properties of the container execution associated with each Batch job. + +Not all the standard container options are supported by AWS Batch. These are the options accepted :: + + + -e, --env string + Set environment variables (format: or =) + --init + Run an init inside the container that forwards signals and reaps processes + --memory-swap int + The total amount of swap memory (in MiB) the container can use: '-1' to enable unlimited swap + --memory-swappiness int + Tune container memory swappiness (0 to 100) (default -1) + --privileged + Give extended privileges to the container + --read-only + Mount the container's root filesystem as read only + --shm-size int + Size (in MiB) of /dev/shm + --tmpfs string + Mount a tmpfs directory (format: :,size=), size is in MiB + -u, --user string + Username or UID (format: [:]) + --ulimit string + Ulimit options (format: =[:]) + +Container options must be passed in their long from for "--option value" or short form "-o value", if available. + +Few examples :: + + containerOptions '--tmpfs /run:rw,noexec,nosuid,size=128 --tmpfs /app:ro,size=64' + + containerOptions '-e MYVAR1 --env MYVAR2=foo2 --env MYVAR3=foo3 --memory-swap 3240000 --memory-swappiness 20 --shm-size 16000000' + + containerOptions '--ulimit nofile=1280:2560 --ulimit nproc=16:32 --privileged' + + +Check the `AWS doc `_ for further details. + Custom AMI ========== There are several reasons why you might need to create your own `AMI (Amazon Machine Image) `_ to use in your Compute environments. Typically: -- you do not want to modify your existing Docker images and prefer to install the CLI tool on the hosting environment +1 - you do not want to modify your existing Docker images and prefer to install the CLI tool on the hosting environment -- the existing AMI (selected from the marketplace) does not have Docker installed +2 - the existing AMI (selected from the marketplace) does not have Docker installed -- you need to attach a larger storage to your EC2 instance (the default ECS instance AMI has only a 30G storage - volume which may not be enough for most data analysis pipelines) +3 - you need to attach a larger storage to your EC2 instance (the default ECS instance AMI has only a 30G storage +volume which may not be enough for most data analysis pipelines) -- you need to install additional software, not available in the Docker image used to execute the job +4 - you need to install additional software, not available in the Docker image used to execute the job Create your custom AMI ---------------------- diff --git a/docs/process.rst b/docs/process.rst index 060af69ef5..45a514007b 100644 --- a/docs/process.rst +++ b/docs/process.rst @@ -1533,7 +1533,7 @@ only for a specific process e.g. mount a custom path:: } -.. warning:: This feature is not supported by :ref:`awsbatch-executor` and :ref:`k8s-executor` executors. +.. warning:: This feature is not supported by :ref:`k8s-executor` and :ref:`azurebatch-executor` executors. .. _process-cpus: diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy index 35de3367d9..0c7c5eee96 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy @@ -17,6 +17,8 @@ package nextflow.processor +import nextflow.util.CmdLineOptionMap + import static nextflow.processor.TaskProcessor.* import java.nio.file.Path @@ -412,9 +414,15 @@ class TaskConfig extends LazyMap implements Cloneable { return opts instanceof CharSequence ? opts.toString() : null } - Map getContainerOptionsMap() { + CmdLineOptionMap getContainerOptionsMap() { def opts = get('containerOptions') - return opts instanceof Map ? opts : Collections.emptyMap() + if( opts instanceof Map ) + return CmdLineOptionMap.fromMap(opts) + if( opts instanceof CharSequence ) + return CmdLineHelper.parseGnuArgs(opts.toString()) + if( opts!=null ) + throw new IllegalArgumentException("Invalid `containerOptions` directive value: $opts [${opts.getClass().getName()}]") + return CmdLineOptionMap.emptyOption() } /** diff --git a/modules/nf-commons/src/main/nextflow/util/CmdLineHelper.groovy b/modules/nf-commons/src/main/nextflow/util/CmdLineHelper.groovy index 080ec62373..b85f119c05 100644 --- a/modules/nf-commons/src/main/nextflow/util/CmdLineHelper.groovy +++ b/modules/nf-commons/src/main/nextflow/util/CmdLineHelper.groovy @@ -16,27 +16,36 @@ */ package nextflow.util + +import groovy.transform.CompileStatic + +import java.util.regex.Pattern + /** - * + * Implement command line parsing helpers + * * @author Paolo Di Tommaso */ +@CompileStatic class CmdLineHelper { - def List args + static private Pattern CLI_OPT = ~/--([a-zA-Z_-]+)(?:\W.*)?$|-([a-zA-Z])(?:\W.*)?$/ + + private List args CmdLineHelper( String cmdLineToBeParsed ) { args = splitter(cmdLineToBeParsed ?: '') } - def boolean contains(String argument) { + private boolean contains(String argument) { return args.indexOf(argument) != -1 } - def getArg( String argument ) { - def pos = args.indexOf(argument) + private getArg( String argument ) { + int pos = args.indexOf(argument) if( pos == -1 ) return null - def result = [] + List result = [] for( int i=pos+1; i + */ +@CompileStatic +@ToString(includes = 'options', includeFields = true) +@EqualsAndHashCode(includes = 'options', includeFields = true) +class CmdLineOptionMap implements CacheFunnel { + + final private Map> options = new LinkedHashMap>() + final private static CmdLineOptionMap EMPTY = new CmdLineOptionMap() + + protected CmdLineOptionMap addOption(String key, String value) { + if ( !options.containsKey(key) ) + options[key] = new ArrayList(10) + options[key].add(value) + return this + } + + boolean hasMultipleValues(String key) { + options.containsKey(key) ? options[key].size() > 1 : false + } + + boolean hasOptions() { + options.size() + } + + List getValues(String key) { + return options.containsKey(key) ? options[key] : Collections.emptyList() as List + } + + def getFirstValue(String key) { + getFirstValueOrDefault(key, null) + } + + boolean asBoolean() { + return options.size()>0 + } + + boolean exists(String key) { + options.containsKey(key) + } + + def getFirstValueOrDefault(String key, String alternative) { + options.containsKey(key) && options[key].get(0) ? options[key].get(0) : alternative + } + + static CmdLineOptionMap fromMap(final Map map) { + def optionMap = new CmdLineOptionMap() + map.each { + optionMap.addOption(it.key as String, it.value as String) + } + return optionMap + } + + static CmdLineOptionMap emptyOption() { + return EMPTY + } + + @Override + String toString() { + def serialized = [] + options.each { + serialized << "option{${it.key}: ${it.value.each {it}}}" + } + return "[${serialized.join(', ')}]" + } + + @Override + Hasher funnel(Hasher hasher, CacheHelper.HashMode mode) { + return CacheHelper.hasher(hasher, options, mode) + } +} diff --git a/modules/nf-commons/src/main/nextflow/util/QuoteStringTokenizer.groovy b/modules/nf-commons/src/main/nextflow/util/QuoteStringTokenizer.groovy index 754cab2e19..32fcaf8cf8 100644 --- a/modules/nf-commons/src/main/nextflow/util/QuoteStringTokenizer.groovy +++ b/modules/nf-commons/src/main/nextflow/util/QuoteStringTokenizer.groovy @@ -26,7 +26,7 @@ package nextflow.util * @author Paolo Di Tommaso * */ -public class QuoteStringTokenizer implements Iterator, Iterable { +class QuoteStringTokenizer implements Iterator, Iterable { private List chars = Arrays.asList(' ' as Character); diff --git a/modules/nf-commons/src/test/nextflow/util/CmdLineHelperTest.groovy b/modules/nf-commons/src/test/nextflow/util/CmdLineHelperTest.groovy new file mode 100644 index 0000000000..f079a635a5 --- /dev/null +++ b/modules/nf-commons/src/test/nextflow/util/CmdLineHelperTest.groovy @@ -0,0 +1,68 @@ +/* + * Copyright 2020-2021, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.util + +import spock.lang.Specification +import spock.lang.Unroll + +/** + * + * @author Paolo Di Tommaso + */ +class CmdLineHelperTest extends Specification{ + + @Unroll + def 'should parse command line' () { + expect: + CmdLineHelper.splitter(SOURCE) == EXPECTED + + where: + SOURCE | EXPECTED + "foo bar baz" | ['foo','bar','baz'] + "foo 'this that'" | ['foo', 'this that'] + } + + @Unroll + def 'should parse args' () { + expect: + CmdLineHelper.parseGnuArgs(SOURCE).toString() == EXPECTED + + where: + SOURCE | EXPECTED + 'a b c' | '[]' + 'a -b c' | '[option{b: [c]}]' + 'a -b -1' | '[option{b: [-1]}]' + '-a b -1' | '[option{a: [b, -1]}]' + "-a 'b -1'" | '[option{a: [b -1]}]' + "-a='b -1'" | '[option{a: [b -1]}]' + '-a "b c"' | '[option{a: [b c]}]' + '-a="b c"' | '[option{a: [b c]}]' + and: + '--foo 1' | '[option{foo: [1]}]' + '--foo 1 --bar 2' | '[option{foo: [1]}, option{bar: [2]}]' + '--foo 1 2 3 --bar 4' | '[option{foo: [1, 2, 3]}, option{bar: [4]}]' + '--foo 1 2 3 --bar' | '[option{foo: [1, 2, 3]}, option{bar: [true]}]' + '--foo --bar' | '[option{foo: [true]}, option{bar: [true]}]' + and: + // single non-gnu is not capture + '--foo 1 -bar 2' | '[option{foo: [1, -bar, 2]}]' + and: + '--foo-name 1 --bar-opt 2' | '[option{foo-name: [1]}, option{bar-opt: [2]}]' + } + +} diff --git a/modules/nf-commons/src/test/nextflow/util/CmdLineOptionMapTest.groovy b/modules/nf-commons/src/test/nextflow/util/CmdLineOptionMapTest.groovy new file mode 100644 index 0000000000..19628b49cf --- /dev/null +++ b/modules/nf-commons/src/test/nextflow/util/CmdLineOptionMapTest.groovy @@ -0,0 +1,89 @@ +package nextflow.util + +import spock.lang.Specification +/** + * + * @author Manuele Simi + */ +class CmdLineOptionMapTest extends Specification { + + def 'test boolean values' () { + + setup: + CmdLineOptionMap options = new CmdLineOptionMap() + options.addOption('iamtrue', 'true') + options.addOption('iamfalse', 'false') + + expect: + options.hasOptions() + options.exists('iamtrue') + options.exists('iamfalse') + options.getFirstValue('iamtrue') == 'true' + options.getFirstValue('iamfalse') == 'false' + options.getFirstValue('idontexist') == null + + } + + def 'test multiple values' () { + + setup: + CmdLineOptionMap options = new CmdLineOptionMap() + options.addOption('firstkey', 'value1') + options.addOption('firstkey', 'value2') + options.addOption('firstkey', 'value3') + options.addOption('secondkey', 'value4') + options.addOption('secondkey', 'value5') + + expect: + options.hasOptions() + options.exists('firstkey') + options.getFirstValue('firstkey') == 'value1' + options.getValues('firstkey').size() == 3 + options.getValues('firstkey').get(0) == 'value1' + options.getValues('firstkey').get(1) == 'value2' + options.getValues('firstkey').get(2) == 'value3' + options.exists('secondkey') + options.getValues('secondkey').size() == 2 + options.getValues('secondkey').get(0) == 'value4' + options.getValues('secondkey').get(1) == 'value5' + + } + + def 'test default values' () { + setup: + CmdLineOptionMap options = new CmdLineOptionMap() + options.addOption('key', '') + options.addOption('key2', null) + + expect: + options.exists('key') + options.exists('key2') + options.getFirstValueOrDefault('key', 'alt' ) == 'alt' + options.getFirstValueOrDefault('key2', 'alt2' ) == 'alt2' + !options.emptyOption().hasOptions() + + } + + def 'should check groovy truth' () { + expect: + // empty => evaluates to false + !new CmdLineOptionMap() + and: + // not empty => evaluates to true + new CmdLineOptionMap().addOption('foo','bar') + } + + def 'should validate equals and hashcode' () { + given: + def map1 = CmdLineOptionMap.fromMap([foo: 'hello']) + def map2 = CmdLineOptionMap.fromMap([foo: 'hello']) + def map3 = CmdLineOptionMap.fromMap([foo: 'world']) + + expect: + map1 == map2 + map1 != map3 + and: + map1.hashCode() == map2.hashCode() + map1.hashCode() != map3.hashCode() + } +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy index 65f4958589..17746dd3c9 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy @@ -62,6 +62,9 @@ import nextflow.processor.TaskRun import nextflow.processor.TaskStatus import nextflow.trace.TraceRecord import nextflow.util.CacheHelper + +import static AwsContainerOptionsMapper.createContainerOpts + /** * Implements a task handler for AWS Batch jobs */ @@ -451,10 +454,14 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler + */ +@CompileStatic +class AwsContainerOptionsMapper { + + static ContainerProperties createContainerOpts(CmdLineOptionMap options) { + final containerProperties = new ContainerProperties() + if ( options?.hasOptions() ) { + checkPrivileged(options, containerProperties) + checkEnvVars(options, containerProperties) + checkUser(options, containerProperties) + checkReadOnly(options, containerProperties) + checkUlimit(options, containerProperties) + LinuxParameters params = checkLinuxParameters(options) + if ( params != null ) + containerProperties.setLinuxParameters(params) + } + return containerProperties + } + + protected static void checkPrivileged(CmdLineOptionMap options, ContainerProperties containerProperties) { + if ( findOptionWithBooleanValue(options, 'privileged') ) + containerProperties.setPrivileged(true); + } + + protected static void checkEnvVars(CmdLineOptionMap options, ContainerProperties containerProperties) { + final keyValuePairs = new ArrayList() + List values = findOptionWithMultipleValues(options, 'env') + values.addAll(findOptionWithMultipleValues(options, 'e')) + values.each { String value -> + final tokens = value.tokenize('=') + keyValuePairs << new KeyValuePair().withName(tokens[0]).withValue(tokens.size() == 2 ? tokens[1] : null) + } + if ( keyValuePairs ) + containerProperties.setEnvironment(keyValuePairs) + } + + protected static void checkUser(CmdLineOptionMap options, ContainerProperties containerProperties) { + String user = findOptionWithSingleValue(options, 'u') + if ( !user) + user = findOptionWithSingleValue(options, 'user') + if ( user ) + containerProperties.setUser(user) + } + + protected static void checkReadOnly(CmdLineOptionMap options, ContainerProperties containerProperties) { + if ( findOptionWithBooleanValue(options, 'read-only') ) + containerProperties.setReadonlyRootFilesystem(true); + } + + protected static void checkUlimit(CmdLineOptionMap options, ContainerProperties containerProperties) { + final ulimits = new ArrayList() + findOptionWithMultipleValues(options, 'ulimit').each { value -> + final tokens = value.tokenize('=') + final limits = tokens[1].tokenize(':') + if ( limits.size() > 1 ) + ulimits << new Ulimit().withName(tokens[0]) + .withSoftLimit(limits[0] as Integer).withHardLimit(limits[1] as Integer) + else + ulimits << new Ulimit().withName(tokens[0]).withSoftLimit(limits[0] as Integer) + } + if ( ulimits.size() ) + containerProperties.setUlimits(ulimits) + } + + protected static LinuxParameters checkLinuxParameters(CmdLineOptionMap options) { + final params = new LinuxParameters() + boolean atLeastOneSet = false + + // shared Memory Size + def value = findOptionWithSingleValue(options, 'shm-size') + if ( value ) { + params.setSharedMemorySize(value as Integer) + atLeastOneSet = true + } + + // tmpfs mounts, e.g --tmpfs /run:rw,noexec,nosuid,size=64 + final tmpfs = new ArrayList() + findOptionWithMultipleValues(options, 'tmpfs').each { ovalue -> + def matcher = ovalue =~ /^(?.*):(?.*?),size=(?.*)$/ + if (matcher.matches()) { + tmpfs << new Tmpfs().withContainerPath(matcher.group('path')) + .withSize(matcher.group('sizeMiB') as Integer) + .withMountOptions(matcher.group('options').tokenize(',')) + } else { + throw new IllegalArgumentException("Found a malformed value '${ovalue}' for --tmpfs option") + } + } + if ( tmpfs ) { + params.setTmpfs(tmpfs) + atLeastOneSet = true + } + + // swap limit equal to memory plus swap + value = findOptionWithSingleValue(options, 'memory-swap') + if ( value ) { + params.setMaxSwap(value as Integer) + atLeastOneSet = true + } + + // run an init inside the container + if ( findOptionWithBooleanValue(options, 'init') ) { + params.setInitProcessEnabled(true) + atLeastOneSet = true + } + + // tune container memory swappiness + value = findOptionWithSingleValue(options, 'memory-swappiness') + if ( value ) { + params.setSwappiness(value as Integer) + atLeastOneSet = true + } + + return atLeastOneSet ? params : null + } + + /** + * Finds the value of an option + * @param name the name of the option + * @return the value, if any, or empty + */ + protected static String findOptionWithSingleValue(CmdLineOptionMap options, String name) { + options.getFirstValueOrDefault(name,null) as String + } + + /** + * Finds the values of an option that can be repeated + * @param name the name of the option + * @return the list of values + */ + protected static List findOptionWithMultipleValues(CmdLineOptionMap options, String name) { + options.getValues(name) + } + + /** + * Checks if a boolean flag exists + * @param name the name of the flag + * @return true if it exists, false otherwise + */ + protected static boolean findOptionWithBooleanValue(CmdLineOptionMap options, String name) { + options.exists(name) ? options.getFirstValue(name) as Boolean : false + } +} diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy index 75a1e5eb86..a88456a13c 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchTaskHandlerTest.groovy @@ -463,7 +463,9 @@ class AwsBatchTaskHandlerTest extends Specification { given: def IMAGE = 'foo/bar:1.0' def JOB_NAME = 'nf-foo-bar-1-0' - def handler = Spy(AwsBatchTaskHandler) + def handler = Spy(AwsBatchTaskHandler) { + getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig) } + } handler.executor = Mock(AwsBatchExecutor) when: @@ -498,7 +500,9 @@ class AwsBatchTaskHandlerTest extends Specification { def JOB_NAME = 'nf-foo-bar-1-0' def executor = Mock(AwsBatchExecutor) def opts = Mock(AwsOptions) - def handler = Spy(AwsBatchTaskHandler) + def handler = Spy(AwsBatchTaskHandler) { + getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig) } + } handler.executor = executor when: @@ -527,7 +531,9 @@ class AwsBatchTaskHandlerTest extends Specification { def JOB_NAME = 'nf-foo-bar-1-0' def opts = Mock(AwsOptions) def executor = Mock(AwsBatchExecutor) - def handler = Spy(AwsBatchTaskHandler) + def handler = Spy(AwsBatchTaskHandler) { + getTask() >> Mock(TaskRun) { getConfig() >> Mock(TaskConfig) } + } handler.executor = executor when: @@ -541,6 +547,31 @@ class AwsBatchTaskHandlerTest extends Specification { result.getContainerProperties().getJobRoleArn() == ROLE } + def 'should set container linux properties' () { + given: + def ROLE = 'aws::foo::bar' + def IMAGE = 'foo/bar:1.0' + def JOB_NAME = 'nf-foo-bar-1-0' + def opts = Mock(AwsOptions) + def taskConfig = new TaskConfig(containerOptions: '--privileged --user foo') + def executor = Mock(AwsBatchExecutor) + def handler = Spy(AwsBatchTaskHandler) { + getTask() >> Mock(TaskRun) { getConfig() >> taskConfig } + } + handler.executor = executor + + when: + def result = handler.makeJobDefRequest(IMAGE) + then: + 1 * handler.normalizeJobDefinitionName(IMAGE) >> JOB_NAME + 1 * handler.getAwsOptions() >> opts + + then: + result.getContainerProperties().getUser() == 'foo' + result.getContainerProperties().getPrivileged() == true + + } + def 'should check task status' () { given: diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsContainerOptionsMapperTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsContainerOptionsMapperTest.groovy new file mode 100644 index 0000000000..5e0278d80d --- /dev/null +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsContainerOptionsMapperTest.groovy @@ -0,0 +1,122 @@ +package nextflow.cloud.aws.batch + +import nextflow.util.CmdLineHelper +import spock.lang.Specification + +/** + * @author Manuele Simi + */ +class AwsContainerOptionsMapperTest extends Specification { + + def 'should set env vars'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--env VAR_FOO -e VAR_FOO2=value2 --env VAR_FOO3=value3') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + def environment = properties.getEnvironment() + environment.size() == 3 + environment.get(0).toString() == '{Name: VAR_FOO,}' + environment.get(1).toString() == '{Name: VAR_FOO3,Value: value3}' + environment.get(2).toString() == '{Name: VAR_FOO2,Value: value2}' + } + + def 'should set ulimits'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--ulimit nofile=1280:2560 --ulimit nproc=16:32') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getUlimits().size() == 2 + properties.getUlimits().get(0).toString() == '{HardLimit: 2560,Name: nofile,SoftLimit: 1280}' + properties.getUlimits().get(1).toString() == '{HardLimit: 32,Name: nproc,SoftLimit: 16}' + + } + + def 'should set user'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--user nf-user') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getUser() == 'nf-user' + } + + def 'should set privileged'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--privileged') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getPrivileged() + } + + def 'should set readonly'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--read-only') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getReadonlyRootFilesystem() + } + + def 'should set tmpfs linux params'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--tmpfs /run:rw,noexec,nosuid,size=64 --tmpfs /app:ro,size=128') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getLinuxParameters().getTmpfs().get(0).toString() == '{ContainerPath: /run,Size: 64,MountOptions: [rw, noexec, nosuid]}' + properties.getLinuxParameters().getTmpfs().get(1).toString() == '{ContainerPath: /app,Size: 128,MountOptions: [ro]}' + } + + def 'should set memory swap '() { + + when: + def map = CmdLineHelper.parseGnuArgs('--memory-swap 2048') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getLinuxParameters().getMaxSwap() == 2048 + } + + def 'should set shared memory size'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--shm-size 12048024') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getLinuxParameters().getSharedMemorySize() == 12048024 + } + + def 'should set memory swappiness'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--memory-swappiness 12048024') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getLinuxParameters().getSwappiness() == 12048024 + } + + def 'should set init'() { + + when: + def map = CmdLineHelper.parseGnuArgs('--init') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getLinuxParameters().getInitProcessEnabled() + } + + def 'should set no params'() { + + when: + def map = CmdLineHelper.parseGnuArgs('') + def properties = AwsContainerOptionsMapper.createContainerOpts(map) + then: + properties.getLinuxParameters() == null + properties.getUlimits() == null + properties.getPrivileged() == null + properties.getReadonlyRootFilesystem() == null + properties.getUser() == null + } +} + diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandler.groovy index 7758400770..56470731d7 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandler.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandler.groovy @@ -346,7 +346,8 @@ class GoogleLifeSciencesTaskHandler extends TaskHandler { req.location = executor.config.location req.cpuPlatform = executor.config.cpuPlatform req.bootDiskSizeGb = executor.config.bootDiskSize?.toGiga() as Integer - req.entryPoint = task.config.getContainerOptionsMap().getOrDefault('entrypoint', GoogleLifeSciencesConfig.DEFAULT_ENTRY_POINT) + task.config.getContainerOptionsMap() + req.entryPoint = task.config.getContainerOptionsMap().getFirstValueOrDefault('entrypoint', GoogleLifeSciencesConfig.DEFAULT_ENTRY_POINT) req.usePrivateAddress = executor.config.usePrivateAddress req.network = executor.config.network req.subnetwork = executor.config.subnetwork diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy index 61535a0736..32729736f9 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy @@ -192,13 +192,6 @@ class GoogleLifeSciencesTaskHandlerTest extends GoogleSpecification { and: req.entryPoint == '/bin/foo' - when: - req = handler.createPipelineRequest() - then: - task.getConfig() >> new TaskConfig(containerOptions: [entrypoint:null]) - and: - req.entryPoint == null - } def 'should create pipeline request/2' () { @@ -260,13 +253,6 @@ class GoogleLifeSciencesTaskHandlerTest extends GoogleSpecification { and: req.entryPoint == '/bin/foo' - when: - req = handler.createPipelineRequest() - then: - task.getConfig() >> new TaskConfig(containerOptions: [entrypoint:null]) - and: - req.entryPoint == null - }