From c6c9bf04f5861a3bc5ec03d86a303148d919e462 Mon Sep 17 00:00:00 2001 From: Keith James Date: Thu, 27 Apr 2023 14:09:47 +0100 Subject: [PATCH] Add choice of run or exec wrapper behaviours Allow the user to choose at install time whether the wrappers generated use singularity run, exec or exec in a service instance. Add logging to syslog for service instance wrappers. --- docker/irods_clients/ubuntu/Dockerfile | 1 + singularity/scripts/singularity-exec-docker | 10 +++++ singularity/scripts/singularity-run-docker | 16 +------- .../scripts/singularity-service-docker | 22 +++++++++++ singularity/scripts/singularity-wrapper | 39 +++++++++++++++---- 5 files changed, 66 insertions(+), 22 deletions(-) create mode 100755 singularity/scripts/singularity-exec-docker create mode 100755 singularity/scripts/singularity-service-docker diff --git a/docker/irods_clients/ubuntu/Dockerfile b/docker/irods_clients/ubuntu/Dockerfile index 1959d5b..ef5f134 100644 --- a/docker/irods_clients/ubuntu/Dockerfile +++ b/docker/irods_clients/ubuntu/Dockerfile @@ -143,6 +143,7 @@ COPY --from=installer /usr/local /usr/local COPY --from=singularity ./scripts/* /usr/local/bin/ COPY --from=installer /opt/docker/irods_clients/manifest.json /usr/local/irods_clients/etc/manifest.json COPY --from=installer /opt/docker/irods_clients/scripts/docker-entrypoint.sh /usr/local/irods_clients/bin/docker-entrypoint.sh +COPY --from=installer /opt/docker/irods_clients/scripts/singularity-* /usr/local/bin/ # Configure the singularity-wrapper script ENV MANIFEST_PATH="/usr/local/irods_clients/etc/manifest.json" diff --git a/singularity/scripts/singularity-exec-docker b/singularity/scripts/singularity-exec-docker new file mode 100755 index 0000000..99d2b87 --- /dev/null +++ b/singularity/scripts/singularity-exec-docker @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +DOCKER_REGISTRY=${DOCKER_REGISTRY:?required, but not set} +DOCKER_USER=${DOCKER_USER:?required, but not set} +DOCKER_IMAGE=${DOCKER_IMAGE:?required, but was not set} +DOCKER_TAG=${DOCKER_TAG:?required, but not set} + +singularity exec "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$@" diff --git a/singularity/scripts/singularity-run-docker b/singularity/scripts/singularity-run-docker index 5b02b0d..b81cb25 100755 --- a/singularity/scripts/singularity-run-docker +++ b/singularity/scripts/singularity-run-docker @@ -1,9 +1,5 @@ #!/bin/bash -# -# Running this script will start an instance of the container, if one is not -# already running, and then execute the command in the instance. -# -# Cleaning up unwanted instances afterwards is outside the scope of this script. + set -euo pipefail DOCKER_REGISTRY=${DOCKER_REGISTRY:?required, but not set} @@ -11,12 +7,4 @@ DOCKER_USER=${DOCKER_USER:?required, but not set} DOCKER_IMAGE=${DOCKER_IMAGE:?required, but was not set} DOCKER_TAG=${DOCKER_TAG:?required, but not set} -# Colons and slashes are not allowed in Singularity instance names -instance="$DOCKER_REGISTRY--$DOCKER_USER--$DOCKER_IMAGE--$DOCKER_TAG" - -if ! singularity instance list | grep "$instance" 2>&1 >/dev/null ; then - singularity instance start \ - "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$instance" 2>&1 >/dev/null -fi - -singularity exec "instance://$instance" "$@" +singularity run "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$@" diff --git a/singularity/scripts/singularity-service-docker b/singularity/scripts/singularity-service-docker new file mode 100755 index 0000000..63f9fe7 --- /dev/null +++ b/singularity/scripts/singularity-service-docker @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Running this script will start an instance of the container, if one is not +# already running, and then execute the command in the instance. +# +# Cleaning up unwanted instances afterwards is outside the scope of this script. +set -euo pipefail + +DOCKER_REGISTRY=${DOCKER_REGISTRY:?required, but not set} +DOCKER_USER=${DOCKER_USER:?required, but not set} +DOCKER_IMAGE=${DOCKER_IMAGE:?required, but was not set} +DOCKER_TAG=${DOCKER_TAG:?required, but not set} + +# Colons and slashes are not allowed in Singularity instance names +instance="$DOCKER_REGISTRY--$DOCKER_USER--$DOCKER_IMAGE--$DOCKER_TAG" + +if ! singularity instance list | grep "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker ; then + singularity instance start \ + "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker +fi + +singularity exec "instance://$instance" "$@" diff --git a/singularity/scripts/singularity-wrapper b/singularity/scripts/singularity-wrapper index 030d155..d6176a0 100755 --- a/singularity/scripts/singularity-wrapper +++ b/singularity/scripts/singularity-wrapper @@ -24,10 +24,22 @@ usage() { Install singularity proxy wrappers for the executables listed in a container's manifest to a nominated directory. -Usage: $0 [-h] [-i ] +Two different types of wrapper are available which differ in how +they run singularity. The default type uses "singularity run", +while the alternative type uses "singularity exec". + +If the "exec" type is used, the wrapper can additionally be made +to create a long-running service instance on first use and +subsequently "exec" within that instance. + +Usage: $0 + [-e] + [-h] + [-i ] [-m ] [-p ] [-r ] + [-s] [-t ] [-u ] [-v] @@ -40,6 +52,8 @@ Operation may be one of: Options: -h Print usage and exit. + -e Use "singularity exec", rather than "singularity run" in the + generated wrappers. -i Docker image name. Required, defaults to the value of the environment variable "\$DOCKER_IMAGE" ("$DOCKER_IMAGE"). -m Manifest file path. Optional, defaults to the value of the @@ -48,6 +62,7 @@ Options: environment variable \$PREFIX ("$PREFIX"). -r Docker registry name. Optional, defaults to the value of the environment variable \$DOCKER_REGISTRY ("$DOCKER_REGISTRY"). + -s Start a long-running service instance (implies use of exec). -t Docker image tag. Optional, defaults to the value of the environment variable \$DOCKER_TAG ("$DOCKER_TAG"). -u Docker user name. Optional, defaults to the value of the @@ -86,7 +101,7 @@ export DOCKER_TAG="$DOCKER_TAG" # time, be permanently set in the installed wrapper. E.g. a candidate # is SINGULARITY_CACHEDIR. -"\$(dirname "\${BASH_SOURCE[0]}")/singularity-run-docker" "$exe" "\$@" +"\$(dirname "\${BASH_SOURCE[0]}")/$singularity_wrap_impl" "$exe" "\$@" EOF chmod +x "$dir/$exe" @@ -96,7 +111,7 @@ chmod +x "$dir/$exe" install_wrappers() { local dir="$PREFIX/bin" install -d "$dir" - cp /usr/local/bin/singularity-run-docker "$PREFIX/bin" + cp "/usr/local/bin/$singularity_wrap_impl" "$PREFIX/bin" for exe in "${wrappers[@]}" ; do write_wrapper "$dir" "$exe" @@ -111,8 +126,13 @@ DOCKER_TAG=${DOCKER_TAG:-latest} PREFIX=${PREFIX:-/opt/wtsi-npg} MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.json"} -while getopts "hi:m:p:r:t:u:v" option; do +singularity_wrap_impl="singularity-run-docker" + +while getopts "hei:m:p:r:st:u:v" option; do case "$option" in + e) + singularity_wrap_impl="singularity-exec-docker" + ;; h) usage exit 0 @@ -129,6 +149,9 @@ while getopts "hi:m:p:r:t:u:v" option; do r) DOCKER_REGISTRY="$OPTARG" ;; + s) + singularity_wrap_impl="singularity-service-docker" + ;; t) DOCKER_TAG="$OPTARG" ;; @@ -156,14 +179,14 @@ fi wrappers=($(jq -j '.executable[] + " "' "$MANIFEST_PATH")) -OPERATION="$@" -if [ -z "$OPERATION" ] ; then +operation="$@" +if [ -z "$operation" ] ; then usage echo -e "\nERROR:\n An operation argument is required" exit 4 fi -case "$OPERATION" in +case "$operation" in list) print_manifest exit 0 @@ -173,7 +196,7 @@ case "$OPERATION" in ;; *) usage - echo -e "\nERROR:\n Invalid wrapper operation '$OPERATION'" + echo -e "\nERROR:\n Invalid wrapper operation '$operation'" exit 4 ;; esac