Feature/irods 4.2.7 client images

docker/Makefile

@@ -292,28 +292,6 @@ ub-18.04-irods-clients-4.2.12.$(TAG): irods_clients/ubuntu/Dockerfile ub-18.04-b
 	--tag $(DOCKER_PREFIX)/ub-18.04-irods-clients-4.2.12:$(TAG) --file $< ./irods_clients
 	touch $@
 
-ub-22.04-irods-clients-4.3-nightly.$(TAG): irods_clients/ubuntu/22.04/Dockerfile ub-22.04-base.$(TAG)
-	docker buildx build $(DOCKER_ARGS) \
-	--load \
-	--build-context singularity=../singularity \
-	--build-arg DOCKER_PREFIX=$(DOCKER_PREFIX) \
-	--build-arg BASE_IMAGE=$(DOCKER_PREFIX)/ub-22.04-base \
-	--build-arg DOCKER_IMAGE=$(subst .$(TAG),,$@) \
-	--build-arg DOCKER_TAG=$(TAG) \
-	--build-arg IRODS_VERSION=4.3-nightly \
-	--build-arg BATON_VERSION=4.1.0 \
-	--build-arg HTSLIB_VERSION=1.18 \
-	--build-arg SAMTOOLS_VERSION=1.18 \
-	--build-arg BCFTOOLS_VERSION=1.18 \
-	--label org.opencontainers.image.title="iRODS 4.3-nightly clients, Ubuntu 22.04" \
-	--label org.opencontainers.image.source=$(git_url) \
-	--label org.opencontainers.image.revision=$(git_commit) \
-	--label org.opencontainers.image.version=$(TAG) \
-	--label org.opencontainers.image.created=$(NOW) \
-	--tag $(DOCKER_PREFIX)/ub-22.04-irods-clients-4.3-nightly:latest \
-	--tag $(DOCKER_PREFIX)/ub-22.04-irods-clients-4.3-nightly:$(TAG) --file $< ./irods_clients
-	touch $@
-
 ub-16.04-irods-clients-dev-4.2.7.$(TAG): irods_clients_dev/ubuntu/16.04/Dockerfile
 	docker buildx build $(DOCKER_ARGS) \
 	--load \
@@ -339,8 +317,6 @@ ub-18.04-irods-clients-dev-4.2.11.$(TAG): irods_clients_dev/ubuntu/Dockerfile
 	--build-arg DOCKER_IMAGE=$(subst .$(TAG),,$@) \
 	--build-arg DOCKER_TAG=$(TAG) \
 	--build-arg IRODS_VERSION=4.2.11 \
-	--build-arg BATON_VERSION=4.1.0 \
-	--build-arg HTSLIB_VERSION=1.18 \
 	--label org.opencontainers.image.title="iRODS 4.2.11 client development, Ubuntu 18.04" \
 	--label org.opencontainers.image.source=$(git_url) \
 	--label org.opencontainers.image.revision=$(git_commit) \

docker/irods_clients/ubuntu/16.04/Dockerfile

@@ -0,0 +1,169 @@
+# syntax=docker/dockerfile:1
+
+ARG BASE_IMAGE=ubuntu:16.04
+FROM $BASE_IMAGE as installer
+
+ARG IRODS_VERSION="4.2.7"
+ARG BATON_VERSION="4.1.0"
+ARG HTSLIB_VERSION="1.18"
+ARG SAMTOOLS_VERSION="1.18"
+ARG BCFTOOLS_VERSION="1.18"
+ARG HTSLIB_PLUGINS_VERSION="201712"
+
+COPY . /opt/docker/irods_clients
+
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-transport-https \
+    apt-utils \
+    ca-certificates \
+    curl \
+    dirmngr \
+    gnupg \
+    gnupg-agent \
+    lsb-release \
+    locales && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8
+
+RUN curl -sSL https://packages.irods.org/irods-signing-key.asc | apt-key add - && \
+    echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-dev="${IRODS_VERSION}" \
+    irods-runtime="${IRODS_VERSION}" \
+    irods-icommands="${IRODS_VERSION}"
+
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "E1DD270288B4E6030699E45FA1715D88E1DF1F24" && \
+    echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/git-core.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    git
+
+# libdeflate-dev is not available on Ubuntu 16.04
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    autoconf \
+    automake \
+    build-essential \
+    less \
+    libtool \
+    pkg-config \
+    python3-sphinx \
+    ssh \
+    libjansson-dev \
+    libbz2-dev \
+    libcurl3-dev \
+    liblzma-dev \
+    zlib1g-dev
+
+ENV CPPFLAGS="-I/usr/include/irods"
+
+RUN cd /tmp && \
+    curl -sSL -O "https://github.com/wtsi-npg/baton/releases/download/${BATON_VERSION}/baton-${BATON_VERSION}.tar.gz" && \
+    tar xfz baton-${BATON_VERSION}.tar.gz && \
+    cd baton-${BATON_VERSION} && \
+    ./configure && \
+    make install
+
+RUN cd /tmp && \
+    curl -sSL -O "https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2" && \
+    tar xfj htslib-${HTSLIB_VERSION}.tar.bz2 && \
+    cd htslib-${HTSLIB_VERSION} && \
+    ./configure --enable-plugins --without-curses && \
+    make install && \
+    ldconfig
+
+RUN cd /tmp && \
+    curl -sSL -O "https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2" && \
+    tar xfj samtools-${SAMTOOLS_VERSION}.tar.bz2 && \
+    cd samtools-${SAMTOOLS_VERSION} && \
+    ./configure --with-htslib=system --without-curses && \
+    make install
+
+RUN cd /tmp && \
+    curl -sSL -O "https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2" && \
+    tar xfj bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
+    cd bcftools-${BCFTOOLS_VERSION} && \
+    ./configure --with-htslib=system --without-curses && \
+    make install
+
+RUN cd /tmp && \
+    git clone --depth 1 --branch ${HTSLIB_PLUGINS_VERSION} "https://github.com/samtools/htslib-plugins.git" && \
+    cd htslib-plugins && \
+    make install
+
+FROM $BASE_IMAGE 
+
+ARG DOCKER_IMAGE
+ARG DOCKER_TAG
+ARG IRODS_VERSION="4.2.7"
+
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-transport-https \
+    apt-utils \
+    ca-certificates \
+    curl \
+    dirmngr \
+    gnupg \
+    gnupg-agent \
+    lsb-release \
+    locales && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8
+
+RUN curl -sSL https://packages.irods.org/irods-signing-key.asc | apt-key add - && \
+    echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-runtime="${IRODS_VERSION}" \
+    irods-icommands="${IRODS_VERSION}"
+
+RUN apt-get install -q -y --no-install-recommends \
+    gnuplot \
+    jq \
+    libjansson4 \
+    libbz2-1.0 \
+    libcurl3 \
+    liblzma5 \
+    perl \
+    zlib1g \
+    unattended-upgrades && \
+    unattended-upgrade -v && \
+    apt-get remove -q -y unattended-upgrades && \
+    apt-get autoremove -q -y && \
+    apt-get clean -q -y && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy the installed baton and samtools applications
+COPY --from=installer /usr/local /usr/local
+
+# Copy the singularity-wrapper scripts
+COPY --from=singularity ./scripts/* /usr/local/bin/
+COPY --from=installer /opt/docker/irods_clients/manifest.json /usr/local/irods_clients/etc/manifest.json
+COPY --from=installer /opt/docker/irods_clients/scripts/docker-entrypoint.sh /usr/local/irods_clients/bin/docker-entrypoint.sh
+
+# Configure the singularity-wrapper script
+ENV MANIFEST_PATH="/usr/local/irods_clients/etc/manifest.json"
+ENV DOCKER_IMAGE="$DOCKER_IMAGE"
+ENV DOCKER_TAG="$DOCKER_TAG"
+
+RUN ldconfig
+
+ENTRYPOINT ["/usr/local/irods_clients/bin/docker-entrypoint.sh"]
+
+CMD ["/bin/bash"]

docker/irods_clients_dev/README.md

@@ -0,0 +1,23 @@
+# Ubuntu iRODS client development
+
+## Summary
+
+An image aimed at development of command line iRODS clients.
+
+ - C development tools, including GCC, GDB, autoconf, automake, libtool
+   pkg-config and valgrind.
+ - [iRODS runtime and development packages](https://github.com/irods/irods)
+ - [iRODS icommands](https://github.com/irods/irods_client_icommands)
+ - [Dependencies required by baton](https://github.com/wtsi-npg/baton)
+
+This image is intended for use anywhere that iRODS C clients are built.
+
+## Usage
+
+This image will create a development container suitable for
+[baton](https://github.com/wtsi-npg/baton) and other iRODS clients.
+
+The Dockerfile supports two build arguments:
+
+- BASE_IMAGE (defaults to "ubuntu:18.04")
+- IRODS_VERSION (defaults to "4.2.11")

docker/irods_clients_dev/ubuntu/16.04/Dockerfile

@@ -0,0 +1,65 @@
+ARG BASE_IMAGE=ubuntu:16.04
+FROM $BASE_IMAGE
+
+ARG IRODS_VERSION="4.2.7"
+
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-transport-https \
+    apt-utils \
+    ca-certificates \
+    curl \
+    dirmngr \
+    gnupg \
+    gnupg-agent \
+    lsb-release \
+    locales && \
+    locale-gen en_GB en_GB.UTF-8 && \
+    localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8
+
+RUN curl -sSL https://packages.irods.org/irods-signing-key.asc | apt-key add - && \
+    echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-dev="${IRODS_VERSION}" \
+    irods-runtime="${IRODS_VERSION}" \
+    irods-icommands="${IRODS_VERSION}"
+
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "E1DD270288B4E6030699E45FA1715D88E1DF1F24" && \
+    echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/git-core.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    git
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    autoconf \
+    automake \
+    build-essential \
+    check \
+    cmake \
+    gdb \
+    jq \
+    lcov \
+    less \
+    libjansson-dev \
+    libtool \
+    netcat \
+    pkg-config \
+    python3-sphinx \
+    ssh \
+    valgrind \
+    unattended-upgrades && \
+    unattended-upgrade -v
+
+ENV CPPFLAGS="-I/usr/include/irods" \
+    CK_DEFAULT_TIMEOUT=20
+
+CMD ["/bin/bash"]

docker/irods_clients_dev/ubuntu/16.04/README.md

@@ -0,0 +1,23 @@
+# Ubuntu iRODS client development
+
+## Summary
+
+An image aimed at development of command line iRODS clients.
+
+ - C development tools, including GCC, GDB, autoconf, automake, libtool
+   pkg-config and valgrind.
+ - [iRODS runtime and development packages](https://github.com/irods/irods)
+ - [iRODS icommands](https://github.com/irods/irods_client_icommands)
+ - [Dependencies required by baton](https://github.com/wtsi-npg/baton)
+
+This image is intended for use anywhere that iRODS C clients are built.
+
+## Usage
+
+This image will create a development container suitable for
+[baton](https://github.com/wtsi-npg/baton) and other iRODS clients.
+
+The Dockerfile supports two build arguments:
+
+- BASE_IMAGE (defaults to "ubuntu:16.04")
+- IRODS_VERSION (defaults to "4.2.7")

singularity/scripts/singularity-wrapper

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Copyright (C) 2023 Genome Research Ltd. All rights reserved.
 #
@@ -17,7 +17,7 @@
 #
 # Author: Keith James <kdj@sanger.ac.uk>
 
-set -euo pipefail
+# set -euo pipefail ## FIXME: bashism
 
 usage() {
     cat 1>&2 << EOF
@@ -36,7 +36,7 @@ Usage: $0
     [-e]
     [-h]
     [-i <Docker image name>]
-    [-m <JSON manifest path>]
+    [-m <manifest path>]
     [-p <wrapper install prefix>]
     [-r <Docker registry name>]
     [-s]
@@ -73,7 +73,7 @@ EOF
 
 # Print an application manifest
 print_manifest() {
-    jq . "$MANIFEST_PATH"
+    cat "$MANIFEST_PATH"
 }
 
 # Write a bash script wrapping an application in a Docker container
@@ -113,9 +113,9 @@ install_wrappers() {
     install -d "$dir"
     cp "/usr/local/bin/$singularity_wrap_impl" "$PREFIX/bin"
 
-    for exe in "${wrappers[@]}" ; do
+    while IFS= read -r exe; do
         write_wrapper "$dir" "$exe"
-    done
+    done < "$MANIFEST_PATH"
 }
 
 DOCKER_REGISTRY=${DOCKER_REGISTRY:-ghcr.io}
@@ -124,7 +124,7 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-""}
 DOCKER_TAG=${DOCKER_TAG:-latest}
 
 PREFIX=${PREFIX:-/opt/wtsi-npg}
-MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.json"}
+MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.txt"}
 
 singularity_wrap_impl="singularity-run-docker"
 
@@ -171,14 +171,11 @@ done
 
 shift $((OPTIND -1))
 
-declare -a wrappers
 if [ ! -e "$MANIFEST_PATH" ] ; then
     echo -e "\nERROR:\n  The manifest of executables at '$MANIFEST_PATH' does not exist"
     exit 4
 fi
 
-wrappers=($(jq -j '.executable[] + " "' "$MANIFEST_PATH"))
-
 operation="$@"
 if [ -z "$operation" ] ; then
     usage