Adding support for Sarus container engine

docs/config.rst

@@ -591,7 +591,7 @@ temp                Mounts a path of your choice as the ``/tmp`` directory in th
 remove              Clean-up the container after the execution (default: ``true``).
 runOptions          This attribute can be used to provide any extra command line options supported by the ``podman run`` command.
 registry            The registry from where container images are pulled. It should be only used to specify a private registry server. It should NOT include the protocol prefix i.e. ``http://``.
-engineOptions       This attribute can be used to provide any option supported by the Docker engine i.e. ``podman [OPTIONS]``.
+engineOptions       This attribute can be used to provide any option supported by the Podman engine i.e. ``podman [OPTIONS]``.
 mountFlags          Add the specified flags to the volume mounts e.g. `mountFlags = 'ro,Z'`
 ================== ================
 
@@ -736,6 +736,28 @@ overwrite           When ``true`` overwrites any existing report file with the s
 ================== ================
 
 
+.. _config-sarus:
+
+Scope `sarus`
+-------------------
+
+The ``sarus`` configuration scope controls how `Sarus <https://sarus.readthedocs.io>`_ containers are executed
+by Nextflow.
+
+The following settings are available:
+
+================== ================
+Name                Description
+================== ================
+enabled             Turn this flag to ``true`` to enable Sarus execution (default: ``false``).
+envWhitelist        Comma separated list of environment variable names to be included in the container environment.
+tty                 Allocates a pseudo-tty (default: ``false``).
+runOptions          This attribute can be used to provide any extra command line options supported by the ``sarus run`` command. For details see: https://sarus.readthedocs.io/en/stable/user/user_guide.html .
+================== ================
+
+Read :ref:`container-sarus` page to learn more about how to use Sarus containers with Nextflow.
+
+
 .. _config-shifter:
 
 Scope `shifter`

docs/container.rst

@@ -315,6 +315,65 @@ Advanced settings
 Podman advanced configuration settings are described in :ref:`config-podman` section in the Nextflow configuration page.
 
 
+.. _container-sarus:
+
+Sarus
+=======
+
+`Sarus <https://sarus.readthedocs.io>`_ is an alternative container runtime to
+Docker. Sarus works by converting Docker images to a common format that can then be
+distributed and launched on HPC systems. The user interface to Sarus enables a user to select an image
+from `Docker Hub <https://hub.docker.com/>`_ and then submit jobs which run entirely within the container.
+
+Prerequisites
+-------------
+
+You need Sarus installed in your execution environment,
+i.e: your personal computer or a distributed cluster, depending
+on where you want to run your pipeline.
+
+.. note:: This feature requires Sarus version 1.5.1 (or later) and Nextflow 23.04.0 (or later).
+
+Images
+------
+
+Sarus converts a docker image to squashfs layers which are distributed and launched in the cluster. For more information on
+how to build Sarus images see the `official documentation <https://sarus.readthedocs.io/en/stable/user/user_guide.html#develop-the-docker-image>`_.
+
+How it works
+------------
+
+The integration for Sarus, at this time, requires you to set up the following parameters in your config file::
+
+  process.container = "dockerhub_user/image_name:image_tag"
+  sarus.enabled = true
+
+and it will always try to search the Docker Hub registry for the images.
+
+.. note:: if you do not specify an image tag, the ``latest`` tag will be fetched by default.
+
+Multiple containers
+-------------------
+
+It is possible to specify a different Sarus image for each process definition in your pipeline script. For example,
+let's suppose you have two processes named ``foo`` and ``bar``. You can specify two different Sarus images
+specifying them in the ``nextflow.config`` file as shown below::
+
+    process {
+        withName:foo {
+            container = 'image_name_1'
+        }
+        withName:bar {
+            container = 'image_name_2'
+        }
+    }
+    sarus {
+        enabled = true
+    }
+
+Read the :ref:`Process scope <config-process>` section to learn more about processes configuration.
+
+
 .. _container-shifter:
 
 Shifter
@@ -328,8 +387,8 @@ from `Docker Hub <https://hub.docker.com/>`_ and then submit jobs which run enti
 Prerequisites
 -------------
 
-You need Shifter and Shifter image gateway installed in your execution environment, i.e: your personal computed or the
-entry node of a distributed cluster. In the case of the distributed cluster case, you should have Shifter installed on
+You need Shifter and Shifter image gateway installed in your execution environment, i.e: your personal computer or the
+entry node of a distributed cluster. In the case of the distributed cluster, you should have Shifter installed on
 all of the compute nodes and the ``shifterimg`` command should also be available and Shifter properly setup to access the
 Image gateway, for more information see the `official documentation <https://github.com/NERSC/shifter/tree/master/doc>`_.
 

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -1163,6 +1163,7 @@ class Session implements ISession {
         def engines = new LinkedList<Map>()
         getContainerConfig0('docker', engines)
         getContainerConfig0('podman', engines)
+        getContainerConfig0('sarus', engines)
         getContainerConfig0('shifter', engines)
         getContainerConfig0('udocker', engines)
         getContainerConfig0('singularity', engines)

modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy

@@ -43,6 +43,8 @@ abstract class ContainerBuilder<V extends ContainerBuilder> {
             return new ApptainerBuilder(containerImage)
         if( engine == 'udocker' )
             return new UdockerBuilder(containerImage)
+        if( engine == 'sarus' )
+            return new SarusBuilder(containerImage)
         if( engine == 'shifter' )
             return new ShifterBuilder(containerImage)
         if( engine == 'charliecloud' )

modules/nextflow/src/main/groovy/nextflow/container/SarusBuilder.groovy

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2020-2022, Seqera Labs
+ * Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.container
+/**
+ * Wrap a task execution in a Sarus container
+ *
+ * See https://sarus.readthedocs.io
+ *
+ * @author Marco De La Pierre <marco.delapierre@gmail.com>
+ */
+class SarusBuilder extends ContainerBuilder<SarusBuilder> {
+
+    private boolean tty
+
+    private boolean verbose
+
+    SarusBuilder( String image ) {
+        assert image
+        this.image = image
+    }
+
+    @Override
+    SarusBuilder build(StringBuilder result) {
+        assert image
+
+        result << 'sarus '
+
+        if( verbose )
+            result << '--verbose '
+
+        result << 'run '
+
+        if( tty )
+            result << '-t '
+
+        // add the environment
+        appendEnv(result)
+
+        // mount the input folders
+        result << makeVolumes(mounts)
+        result << '-w "$PWD" '
+
+        if( runOptions )
+            result << runOptions.join(' ') << ' '
+
+        // finally the container name
+        result << image
+
+        runCommand = result.toString()
+        return this
+    }
+
+    SarusBuilder params( Map params ) {
+
+        if( params.containsKey('verbose') )
+            this.verbose = params.verbose.toString() == 'true'
+
+        if( params.containsKey('entry') )
+            this.entryPoint = params.entry
+
+        if( params.containsKey('runOptions') )
+            addRunOptions(params.runOptions.toString())
+
+        if( params.containsKey('tty') )
+            this.tty = params.tty?.toString() == 'true'
+
+        return this
+    }
+
+    @Override
+    String getRunCommand() {
+        def run = super.getRunCommand()
+        def result = """\
+        sarus pull $image 1>&2
+        """.stripIndent()
+        result += run
+        return result
+    }
+
+    @Override
+    protected String composeVolumePath( String path, boolean readOnly = false ) {
+        return "--mount=type=bind,source=${escape(path)},destination=${escape(path)}"
+    }
+}

modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy

@@ -141,7 +141,7 @@ class BashWrapperBuilder {
     }
 
     protected boolean fixOwnership() {
-        systemOsName == 'Linux' && containerConfig?.fixOwnership && runWithContainer && containerConfig.engine == 'docker' // <-- note: only for docker (shifter is not affected)
+        systemOsName == 'Linux' && containerConfig?.fixOwnership && runWithContainer && containerConfig.engine == 'docker' // <-- note: only for docker (sarus and shifter are not affected)
     }
 
     protected isMacOS() {

modules/nextflow/src/test/groovy/nextflow/SessionTest.groovy

@@ -369,6 +369,7 @@ class SessionTest extends Specification {
         'podman'       | [enabled: true, x:'alpha', y: 'beta']
         'podman'       | [enabled: true, x:'alpha', y: 'beta', registry: 'd.reg']
         'udocker'      | [enabled: true, x:'alpha', y: 'beta']
+        'sarus'        | [enabled: true, x:'delta', y: 'gamma']
         'shifter'      | [enabled: true, x:'delta', y: 'gamma']
         'singularity'  | [enabled: true, x:'delta', y: 'gamma']
         'charliecloud' | [enabled: true, x:'delta', y: 'gamma']

modules/nextflow/src/test/groovy/nextflow/container/ContainerBuilderTest.groovy

@@ -83,6 +83,7 @@ class ContainerBuilderTest extends Specification {
         'podman'            | PodmanBuilder
         'singularity'       | SingularityBuilder
         'apptainer'         | ApptainerBuilder
+        'sarus'             | SarusBuilder
         'shifter'           | ShifterBuilder
         'charliecloud'      | CharliecloudBuilder
         'udocker'           | UdockerBuilder

modules/nextflow/src/test/groovy/nextflow/container/SarusBuilderTest.groovy

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020-2022, Seqera Labs
+ * Copyright 2013-2019, Centre for Genomic Regulation (CRG)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.container
+import spock.lang.Specification
+import java.nio.file.Paths
+/**
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+class SarusrBuilderTest extends Specification {
+
+    def 'test sarus env'() {
+
+        given:
+        def builder = new SarusBuilder('x')
+
+        expect:
+        builder.makeEnv('X=1').toString() == '-e "X=1"'
+        builder.makeEnv([VAR_X:1, VAR_Y: 2]).toString() == '-e "VAR_X=1" -e "VAR_Y=2"'
+    }
+
+    def 'should build the sarus run command' () {
+
+        expect:
+        new SarusBuilder('busybox')
+                .build()
+                .@runCommand == 'sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" busybox'
+
+        new SarusBuilder('busybox')
+                .params(verbose: true)
+                .build()
+                .@runCommand == 'sarus --verbose run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" busybox'
+
+        new SarusBuilder('fedora')
+                .addEnv([VAR_X:1, VAR_Y:2])
+                .addEnv("VAR_Z=3")
+                .build()
+                .@runCommand == 'sarus run -e "VAR_X=1" -e "VAR_Y=2" -e "VAR_Z=3" --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" fedora'
+
+        new SarusBuilder('busybox')
+                .params(runOptions: '-x --zeta')
+                .build()
+                .@runCommand == 'sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" -x --zeta busybox'
+
+        new SarusBuilder('fedora')
+                .addEnv([VAR_X:1, VAR_Y:2])
+                .addMount(Paths.get('/home/db'))
+                .addMount(Paths.get('/home/db'))  // <-- add twice the same to prove that the final string won't contain duplicates
+                .build()
+                .@runCommand == 'sarus run -e "VAR_X=1" -e "VAR_Y=2" --mount=type=bind,source=/home/db,destination=/home/db --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" fedora'
+
+    }
+
+    def 'should get run command line' () {
+
+        when:
+        def cli = new SarusBuilder('ubuntu:14').build().getRunCommand()
+        then:
+        cli ==  '''\
+        sarus pull ubuntu:14 1>&2
+        sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" ubuntu:14
+        '''
+        .stripIndent().trim()
+
+        when:
+        cli = new SarusBuilder('ubuntu:14').build().getRunCommand('bwa --this --that file.fasta')
+        then:
+        cli ==  '''\
+        sarus pull ubuntu:14 1>&2
+        sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" ubuntu:14 bwa --this --that file.fasta
+        '''
+        .stripIndent().trim()
+
+        when:
+        cli = new SarusBuilder('ubuntu:14').params(entry:'/bin/bash').build().getRunCommand('bwa --this --that file.fasta')
+        then:
+        cli ==  '''\
+        sarus pull ubuntu:14 1>&2
+        sarus run --mount=type=bind,source="$PWD",destination="$PWD" -w "$PWD" ubuntu:14 /bin/bash -c "bwa --this --that file.fasta"
+        '''
+        .stripIndent().trim()
+
+    }
+
+
+
+}