diff --git a/docs/config.rst b/docs/config.rst index 3be4e6b518..ba039a05e3 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -591,7 +591,7 @@ temp Mounts a path of your choice as the ``/tmp`` directory in th remove Clean-up the container after the execution (default: ``true``). runOptions This attribute can be used to provide any extra command line options supported by the ``podman run`` command. registry The registry from where container images are pulled. It should be only used to specify a private registry server. It should NOT include the protocol prefix i.e. ``http://``. -engineOptions This attribute can be used to provide any option supported by the Docker engine i.e. ``podman [OPTIONS]``. +engineOptions This attribute can be used to provide any option supported by the Podman engine i.e. ``podman [OPTIONS]``. mountFlags Add the specified flags to the volume mounts e.g. `mountFlags = 'ro,Z'` ================== ================ @@ -736,6 +736,28 @@ overwrite When ``true`` overwrites any existing report file with the s ================== ================ +.. _config-sarus: + +Scope `sarus` +------------------- + +The ``sarus`` configuration scope controls how `Sarus `_ containers are executed +by Nextflow. + +The following settings are available: + +================== ================ +Name Description +================== ================ +enabled Turn this flag to ``true`` to enable Sarus execution (default: ``false``). +envWhitelist Comma separated list of environment variable names to be included in the container environment. +tty Allocates a pseudo-tty (default: ``false``). +runOptions This attribute can be used to provide any extra command line options supported by the ``sarus run`` command. For details see: https://sarus.readthedocs.io/en/stable/user/user_guide.html . +================== ================ + +Read :ref:`container-sarus` page to learn more about how to use Sarus containers with Nextflow. + + .. _config-shifter: Scope `shifter` diff --git a/docs/container.rst b/docs/container.rst index a901ce06ca..8a353aa37a 100644 --- a/docs/container.rst +++ b/docs/container.rst @@ -315,6 +315,65 @@ Advanced settings Podman advanced configuration settings are described in :ref:`config-podman` section in the Nextflow configuration page. +.. _container-sarus: + +Sarus +======= + +`Sarus `_ is an alternative container runtime to +Docker. Sarus works by converting Docker images to a common format that can then be +distributed and launched on HPC systems. The user interface to Sarus enables a user to select an image +from `Docker Hub `_ and then submit jobs which run entirely within the container. + +Prerequisites +------------- + +You need Sarus installed in your execution environment, +i.e: your personal computer or a distributed cluster, depending +on where you want to run your pipeline. + +.. note:: This feature requires Sarus version 1.5.1 (or later) and Nextflow 22.12.0-edge (or later). + +Images +------ + +Sarus converts a docker image to squashfs layers which are distributed and launched in the cluster. For more information on +how to build Sarus images see the `official documentation `_. + +How it works +------------ + +The integration for Sarus, at this time, requires you to set up the following parameters in your config file:: + + process.container = "dockerhub_user/image_name:image_tag" + sarus.enabled = true + +and it will always try to search the Docker Hub registry for the images. + +.. note:: if you do not specify an image tag, the ``latest`` tag will be fetched by default. + +Multiple containers +------------------- + +It is possible to specify a different Sarus image for each process definition in your pipeline script. For example, +let's suppose you have two processes named ``foo`` and ``bar``. You can specify two different Sarus images +specifying them in the ``nextflow.config`` file as shown below:: + + process { + withName:foo { + container = 'image_name_1' + } + withName:bar { + container = 'image_name_2' + } + } + sarus { + enabled = true + } + +Read the :ref:`Process scope ` section to learn more about processes configuration. + + .. _container-shifter: Shifter @@ -328,8 +387,8 @@ from `Docker Hub `_ and then submit jobs which run enti Prerequisites ------------- -You need Shifter and Shifter image gateway installed in your execution environment, i.e: your personal computed or the -entry node of a distributed cluster. In the case of the distributed cluster case, you should have Shifter installed on +You need Shifter and Shifter image gateway installed in your execution environment, i.e: your personal computer or the +entry node of a distributed cluster. In the case of the distributed cluster, you should have Shifter installed on all of the compute nodes and the ``shifterimg`` command should also be available and Shifter properly setup to access the Image gateway, for more information see the `official documentation `_. diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 9fc4248502..b628b4c398 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -1163,6 +1163,7 @@ class Session implements ISession { def engines = new LinkedList() getContainerConfig0('docker', engines) getContainerConfig0('podman', engines) + getContainerConfig0('sarus', engines) getContainerConfig0('shifter', engines) getContainerConfig0('udocker', engines) getContainerConfig0('singularity', engines) diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy index 1a9c25eec3..ab6b980f09 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy @@ -43,6 +43,8 @@ abstract class ContainerBuilder { return new ApptainerBuilder(containerImage) if( engine == 'udocker' ) return new UdockerBuilder(containerImage) + if( engine == 'sarus' ) + return new SarusBuilder(containerImage) if( engine == 'shifter' ) return new ShifterBuilder(containerImage) if( engine == 'charliecloud' ) diff --git a/modules/nextflow/src/main/groovy/nextflow/container/SarusBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/SarusBuilder.groovy new file mode 100644 index 0000000000..41ab3136e5 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/container/SarusBuilder.groovy @@ -0,0 +1,98 @@ +/* + * Copyright 2022, Pawsey Supercomputing Research Centre + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.container +/** + * Wrap a task execution in a Sarus container + * + * See https://sarus.readthedocs.io + * + * @author Marco De La Pierre + */ +class SarusBuilder extends ContainerBuilder { + + private boolean tty + + private boolean verbose + + SarusBuilder( String image ) { + assert image + this.image = image + } + + @Override + SarusBuilder build(StringBuilder result) { + assert image + + result << 'sarus ' + + if( verbose ) + result << '--verbose ' + + result << 'run ' + + if( tty ) + result << '-t ' + + // add the environment + appendEnv(result) + + // mount the input folders + result << makeVolumes(mounts) + result << '-w "$PWD" ' + + if( runOptions ) + result << runOptions.join(' ') << ' ' + + // finally the container name + result << image + + runCommand = result.toString() + return this + } + + SarusBuilder params( Map params ) { + + if( params.containsKey('verbose') ) + this.verbose = params.verbose.toString() == 'true' + + if( params.containsKey('entry') ) + this.entryPoint = params.entry + + if( params.containsKey('runOptions') ) + addRunOptions(params.runOptions.toString()) + + if( params.containsKey('tty') ) + this.tty = params.tty?.toString() == 'true' + + return this + } + + @Override + String getRunCommand() { + def run = super.getRunCommand() + def result = """\ + sarus pull $image 1>&2 + """.stripIndent() + result += run + return result + } + + @Override + protected String composeVolumePath( String path, boolean readOnly = false ) { + return "--mount=type=bind,source=${escape(path)},destination=${escape(path)}" + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index 8ae0a89fd2..bd0676e76b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -141,7 +141,7 @@ class BashWrapperBuilder { } protected boolean fixOwnership() { - systemOsName == 'Linux' && containerConfig?.fixOwnership && runWithContainer && containerConfig.engine == 'docker' // <-- note: only for docker (shifter is not affected) + systemOsName == 'Linux' && containerConfig?.fixOwnership && runWithContainer && containerConfig.engine == 'docker' // <-- note: only for docker (other container runtimes are not affected) } protected isMacOS() { diff --git a/modules/nextflow/src/test/groovy/nextflow/SessionTest.groovy b/modules/nextflow/src/test/groovy/nextflow/SessionTest.groovy index 7c2add3681..7a07722fe3 100644 --- a/modules/nextflow/src/test/groovy/nextflow/SessionTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/SessionTest.groovy @@ -369,6 +369,7 @@ class SessionTest extends Specification { 'podman' | [enabled: true, x:'alpha', y: 'beta'] 'podman' | [enabled: true, x:'alpha', y: 'beta', registry: 'd.reg'] 'udocker' | [enabled: true, x:'alpha', y: 'beta'] + 'sarus' | [enabled: true, x:'delta', y: 'gamma'] 'shifter' | [enabled: true, x:'delta', y: 'gamma'] 'singularity' | [enabled: true, x:'delta', y: 'gamma'] 'charliecloud' | [enabled: true, x:'delta', y: 'gamma'] diff --git a/modules/nextflow/src/test/groovy/nextflow/container/ContainerBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/container/ContainerBuilderTest.groovy index 6f9c1e865c..69476c27da 100644 --- a/modules/nextflow/src/test/groovy/nextflow/container/ContainerBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/container/ContainerBuilderTest.groovy @@ -83,6 +83,7 @@ class ContainerBuilderTest extends Specification { 'podman' | PodmanBuilder 'singularity' | SingularityBuilder 'apptainer' | ApptainerBuilder + 'sarus' | SarusBuilder 'shifter' | ShifterBuilder 'charliecloud' | CharliecloudBuilder 'udocker' | UdockerBuilder diff --git a/modules/nextflow/src/test/groovy/nextflow/container/SarusBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/container/SarusBuilderTest.groovy new file mode 100644 index 0000000000..0082287614 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/container/SarusBuilderTest.groovy @@ -0,0 +1,101 @@ +/* + * Copyright 2022, Pawsey Supercomputing Research Centre + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.container +import spock.lang.Specification +import java.nio.file.Paths +/** + * + * @author Marco De La Pierre + */ +class SarusrBuilderTest extends Specification { + + def 'test sarus env'() { + + given: + def builder = new SarusBuilder('x') + + expect: + builder.makeEnv('X=1').toString() == '-e "X=1"' + builder.makeEnv([VAR_X:1, VAR_Y: 2]).toString() == '-e "VAR_X=1" -e "VAR_Y=2"' + } + + def 'should build the sarus run command' () { + + expect: + new SarusBuilder('busybox') + .build() + .@runCommand == 'sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" busybox' + + new SarusBuilder('busybox') + .params(verbose: true) + .build() + .@runCommand == 'sarus --verbose run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" busybox' + + new SarusBuilder('fedora') + .addEnv([VAR_X:1, VAR_Y:2]) + .addEnv("VAR_Z=3") + .build() + .@runCommand == 'sarus run -e "VAR_X=1" -e "VAR_Y=2" -e "VAR_Z=3" --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" fedora' + + new SarusBuilder('busybox') + .params(runOptions: '-x --zeta') + .build() + .@runCommand == 'sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" -x --zeta busybox' + + new SarusBuilder('fedora') + .addEnv([VAR_X:1, VAR_Y:2]) + .addMount(Paths.get('/home/db')) + .addMount(Paths.get('/home/db')) // <-- add twice the same to prove that the final string won't contain duplicates + .build() + .@runCommand == 'sarus run -e "VAR_X=1" -e "VAR_Y=2" --mount=type=bind,source=/home/db,destination=/home/db --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" fedora' + + } + + def 'should get run command line' () { + + when: + def cli = new SarusBuilder('ubuntu:14').build().getRunCommand() + then: + cli == '''\ + sarus pull ubuntu:14 1>&2 + sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" ubuntu:14 + ''' + .stripIndent().trim() + + when: + cli = new SarusBuilder('ubuntu:14').build().getRunCommand('bwa --this --that file.fasta') + then: + cli == '''\ + sarus pull ubuntu:14 1>&2 + sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" ubuntu:14 bwa --this --that file.fasta + ''' + .stripIndent().trim() + + when: + cli = new SarusBuilder('ubuntu:14').params(entry:'/bin/bash').build().getRunCommand('bwa --this --that file.fasta') + then: + cli == '''\ + sarus pull ubuntu:14 1>&2 + sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" ubuntu:14 /bin/bash -c "bwa --this --that file.fasta" + ''' + .stripIndent().trim() + + } + + + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index f2f7f4acc4..275841f10e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -806,6 +806,81 @@ class BashWrapperBuilderTest extends Specification { binding.cleanup_cmd == 'docker rm $NXF_BOXID &>/dev/null || true\n' } + def 'should create wrapper with sarus'() { + when: + def binding = newBashWrapperBuilder( + containerEnabled: true, + containerImage: 'busybox', + containerConfig: [enabled: true, engine: 'sarus'] as ContainerConfig ).makeBinding() + + then: + binding.launch_cmd == '''\ + sarus pull busybox 1>&2 + sarus run --mount=type=bind,source=/work/dir,destination=/work/dir -w "$PWD" busybox /bin/bash -ue /work/dir/.command.sh + '''.stripIndent().rightTrim() + binding.cleanup_cmd == "" + binding.kill_cmd == '[[ "$pid" ]] && kill $pid 2>/dev/null' + } + + def 'should create wrapper with sarus and environment'() { + when: + def binding = newBashWrapperBuilder( + containerEnabled: true, + containerImage: 'busybox', + environment: [FOO: 'xxx'], + containerConfig: [enabled: true, engine: 'sarus'] as ContainerConfig ).makeBinding() + + then: + binding.launch_cmd == '''\ + sarus pull busybox 1>&2 + sarus run --mount=type=bind,source=/work/dir,destination=/work/dir -w "$PWD" busybox /bin/bash -c "eval $(nxf_container_env); /bin/bash -ue /work/dir/.command.sh" + '''.stripIndent().rightTrim() + binding.cleanup_cmd == "" + binding.kill_cmd == '[[ "$pid" ]] && kill $pid 2>/dev/null' + and: + binding.container_env == '''\ + nxf_container_env() { + cat << EOF + export FOO="xxx" + EOF + } + '''.stripIndent() + } + + def 'should create wrapper with sarus with mount'() { + when: + def binding = newBashWrapperBuilder( + containerEnabled: true, + containerImage: 'busybox', + containerMount: '/folder with blanks' as Path, + containerConfig: [enabled: true, engine: 'sarus'] as ContainerConfig ).makeBinding() + + then: + binding.launch_cmd == '''\ + sarus pull busybox 1>&2 + sarus run --mount=type=bind,source=/folder\\ with\\ blanks,destination=/folder\\ with\\ blanks --mount=type=bind,source=/work/dir,destination=/work/dir -w "$PWD" busybox /bin/bash -ue /work/dir/.command.sh + '''.stripIndent().rightTrim() + binding.cleanup_cmd == "" + binding.kill_cmd == '[[ "$pid" ]] && kill $pid 2>/dev/null' + } + + def 'should create wrapper with sarus container custom options'() { + when: + def binding = newBashWrapperBuilder( + containerEnabled: true, + containerImage: 'busybox', + containerOptions: '--mount=type=bind,source=/foo,destination=/bar', + containerConfig: [enabled: true, engine: 'sarus'] as ContainerConfig ).makeBinding() + + then: + binding.launch_cmd == '''\ + sarus pull busybox 1>&2 + sarus run --mount=type=bind,source=/work/dir,destination=/work/dir -w "$PWD" --mount=type=bind,source=/foo,destination=/bar busybox /bin/bash -ue /work/dir/.command.sh + '''.stripIndent().rightTrim() + binding.cleanup_cmd == "" + binding.kill_cmd == '[[ "$pid" ]] && kill $pid 2>/dev/null' + } + def 'should create wrapper with shifter'() { when: def binding = newBashWrapperBuilder( diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy index 93e3862f07..4c5e27269a 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy @@ -332,6 +332,7 @@ class TaskRunTest extends Specification { 'docker' | 'busybox' | [enabled: true, x:'alpha', y: 'beta'] 'docker' | 'd.reg/busybox' | [enabled: true, x:'alpha', y: 'beta', registry: 'd.reg'] 'udocker' | 'busybox:latest' | [enabled: true, x:'alpha', y: 'beta'] + 'sarus' | 'busybox' | [enabled: true, x:'delta', y: 'gamma'] 'shifter' | 'docker:busybox:latest' | [enabled: true, x:'delta', y: 'gamma'] }