Merge devel to master for release 5.2

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/create-release.yml

@@ -5,17 +5,21 @@ on:
     tags:
       - "*"
 
+env:
+  BUNDLE_NAME: bundle-ub-16.04-irods-clients-4.2.7.tgz
+
 jobs:
-  release:
+
+  release_variables:
     runs-on: ubuntu-latest
-    
+
     defaults:
       run:
         shell: bash -l -e -o pipefail {0}
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -24,36 +28,31 @@ jobs:
         run: |
           # Avoid git exiting when Actions runs
           git config --global --add safe.directory "$PWD"
-          
+
           git fetch --tags --force
 
       - name: "Get release variables"
         run: |
           echo 'RELEASE_VERSION='$(git describe --always --tags) >> $GITHUB_ENV
           echo 'MASTER_SHA='$(git rev-parse origin/master) >> $GITHUB_ENV
 
-      - name: "Create Release"
-        uses: ncipollo/release-action@v1.12.0
-        with:
-          name: ${{ env.RELEASE_VERSION }}
-          prerelease: ${{ !(github.sha == env.MASTER_SHA) }}
-          generateReleaseNotes: true
-
     outputs:
       isRelease: ${{ github.sha == env.MASTER_SHA }}
+      preRelease: ${{ !(github.sha == env.MASTER_SHA) }}
+      releaseVersion: ${{ env.RELEASE_VERSION }}
 
   deploy:
     runs-on: ubuntu-latest
 
-    needs: release
+    needs: release_variables
 
     # Workaround for https://github.com/actions/runner/issues/1483
     # Actions coerces boolean to string
-    if: needs.release.outputs.isRelease == 'true'
+    if: needs.release_variables.outputs.isRelease == 'true'
 
     steps:
       - name: "Free disk space on the runner"
-        uses: jlumbroso/free-disk-space@f68fdb76e2ea636224182cfb7377ff9a1708f9b8  # v1.3.0
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be  # v1.3.1
         with:
           tool-cache: false
           android: true
@@ -64,7 +63,7 @@ jobs:
           swap-storage: false
 
       - name: "Checkout code"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -77,12 +76,12 @@ jobs:
           git fetch --tags --force
 
       - name: "Set up Docker Buildx"
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           buildkitd-flags: --debug
 
       - name: "Login to ghcr.io"
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
             registry: ghcr.io
             username: ${{ github.repository_owner }}
@@ -103,3 +102,59 @@ jobs:
             docker images
 
             make push GITHUB_ACTIONS=${GITHUB_ACTIONS} GITHUB_REPOSITORY_OWNER=${{ github.repository_owner }}
+
+  bundle:
+    runs-on: ubuntu-latest
+    container: "ghcr.io/wtsi-npg/ub-16.04-irods-clients-4.2.7:latest"
+
+    needs: deploy
+
+    if: always()
+
+    steps:
+    - name: "Install exodus"
+      run: |
+        apt-get update
+        apt-get install -q -y python3-pip
+        pip3 install exodus-bundler
+
+    - name: "Bundle clients"
+
+      run: exodus --tarball --add /usr/lib/irods/plugins/ --add /lib/x86_64-linux-gnu/ --output $BUNDLE_NAME /usr/local/bin/baton-* /usr/bin/ils /usr/bin/iinit /usr/bin/imeta
+
+    - name: "Upload bundle"
+      uses: actions/upload-artifact@v3
+      with:
+        name: client_bundle
+        path: ${{ env.BUNDLE_NAME }}
+
+  release:
+    runs-on: ubuntu-latest
+
+    needs: [release_variables, bundle]
+
+    if: always()
+
+    defaults:
+      run:
+        shell: bash -l -e -o pipefail {0}
+
+    steps:
+
+      - name: "Download artifact"
+        uses: actions/download-artifact@v4
+        with:
+          name: client_bundle
+
+      - name: "Create Release"
+        uses: ncipollo/release-action@v1.14.0
+        with:
+          name: ${{ env.NAME }}
+          prerelease: ${{ env.PRERELEASE }}
+          artifacts: ${{ env.BUNDLE_NAME }}
+          removeArtifacts: true
+          artifactErrorsFailBuild: true
+          generateReleaseNotes: true
+        env:
+          NAME: ${{ needs.release_variables.outputs.releaseVersion }}
+          PRERELEASE: ${{ needs.release_variables.outputs.preRelease }}

.github/workflows/nightly-build.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -31,12 +31,12 @@ jobs:
           git fetch --tags --force
 
       - name: "Set up Docker Buildx"
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           buildkitd-flags: --debug
 
       - name: "Login to ghcr.io"
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
             registry: ghcr.io
             username: ${{ github.repository_owner }}

README.md

@@ -73,6 +73,22 @@ e.g. Install wrappers to $PREFIX/bin:
     ...
     -rwxr-xr-x 1 kdj staff 409 Apr 12 15:47 samtools
 
+# iRODS Client Bundle
+
+The release contains irods 4.2.7 clients bundled from
+`ub-16.04-irods-clients-4.2.7` to be used on ONT machines, where we
+cannot run containers.  Given the complexity of the iRODS clients'
+dependency chain, the location of the plugins must be provided in the
+iRODS environment file using the `irods_plugins_home` key. Some
+dependencies for these plugins must also be preloaded to avoid local
+system libraries from being pulled in, e.g.:
+
+    $ LD_PRELOAD=path/libarchive.so.16:path/libcrypto.so.1.0.0:path/libssl.so.1.0.0:path/libxml2.so.2:path/libicuuc.so.55:path/libicudata.so.55 \
+        ./exodus/bin/iinit
+
+A script to do this is set up by the ansible used to set up the instruments
+(https://gitlab.internal.sanger.ac.uk/npg/oxford_nanopore_instruments).
+
 
 ## Author
 

singularity/scripts/singularity-service-docker

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Running this script will start an instance of the container, if one is not 
+# Running this script will start an instance of the container, if one is not
 # already running, and then execute the command in the instance.
 #
 # Cleaning up unwanted instances afterwards is outside the scope of this script.
@@ -11,12 +11,45 @@ DOCKER_USER=${DOCKER_USER:?required, but not set}
 DOCKER_IMAGE=${DOCKER_IMAGE:?required, but was not set}
 DOCKER_TAG=${DOCKER_TAG:?required, but not set}
 
+DOCKER_URL="docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG"
+
 # Colons and slashes are not allowed in Singularity instance names
 instance="$DOCKER_REGISTRY--$DOCKER_USER--$DOCKER_IMAGE--$DOCKER_TAG"
 
-if ! singularity instance list | grep "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker ; then
-    singularity instance start \
-        "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker
+# Requires bash >=4.1 for file descriptor management with "{<variable>}"
+LOCK_FILE="/var/lock/$instance--$USER.lock"
+LOCK_TIMEOUT=60
+
+# Get the first free file descriptor
+exec {FD}>"$LOCK_FILE"
+
+# Release the lock and close the file descriptor
+clean_up() {
+    flock --unlock $FD || {
+        echo >&2 "Failed to release lock on $LOCK_FILE (fd: $FD): $?"
+    }
+    exec {FD}>&-
+}
+
+clean_up_and_exit() {
+    clean_up
+    exit 1
+}
+
+# clean_up_and_exit on these signals
+trap clean_up_and_exit SIGINT SIGTERM SIGUSR1 SIGUSR2
+
+flock --exclusive --timeout $LOCK_TIMEOUT $FD || {
+    echo >&2 "Failed to obtain lock on $LOCK_FILE (fd: $FD): $?"
+    exit 1
+}
+
+if ! singularity instance list 2> >(logger -s -p user.err -t singularity-service-docker) |\
+        grep "$instance" 2>&1 >/dev/null
+then
+    singularity --silent instance start "$DOCKER_URL" "$instance" 2> >(logger -s -p user.err -t singularity-service-docker) >/dev/null
 fi
 
+clean_up
+
 singularity exec "instance://$instance" "$@"