Merge pull request #21 from wtsi-npg/devel

Merge devel into master to create release 1.2

.github/workflows/pushing_container_image.yml

@@ -0,0 +1,49 @@
+name: Publish image
+on:
+  release:
+    types: [published]
+jobs:
+  push_to_registries:
+    name: Push image to multiple registries
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          buildkitd-flags: --debug
+      -
+        name: Login to ghcr.io
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: lowercase Repo
+        run: |
+            echo "REPO_OWNER=${GITHUB_REPO_OWNER,,}" >>${GITHUB_ENV}
+        env:
+          GITHUB_REPO_OWNER: ${{ github.repository_owner }}
+
+
+      - name: Build
+        run: |
+            cd docker
+            make REPO_OWNER=${{ env.REPO_OWNER }}
+     
+      - name: push images
+        run: |      
+            docker images
+            export TAG=$(git describe --always)
+            docker push ghcr.io/${{ env.REPO_OWNER }}/ub-16.04-base:$TAG
+            docker push ghcr.io/${{ env.REPO_OWNER }}/ub-18.04-base:$TAG
+            docker push ghcr.io/${{ env.REPO_OWNER }}/ub-16.04-conda:$TAG
+            docker push ghcr.io/${{ env.REPO_OWNER }}/ub-16.04-irods-4.2.7:$TAG
+            docker push ghcr.io/${{ env.REPO_OWNER }}/ub-18.04-irods-4.2.8:$TAG
+

README.md

@@ -2,11 +2,6 @@
 
 ## The images ##
 
-### base/ubuntu/12.04 ###
-
-A base image with curl and gosu installed, intended for multi-stage
-builds where curl is later dropped.
-
 ### base/ubuntu/16.04 ###
 
 A base image with curl and gosu installed, intended for multi-stage
@@ -17,16 +12,19 @@ builds where curl is later dropped.
 A base image with curl and gosu installed, intended for multi-stage
 builds where curl is later dropped.
 
-### conda/ubuntu/12.04 ###
+### base/centos/7 ###
+
+A base image with gosu installed.
+
+### conda/ubuntu/16.04 ###
 
 A Conda package-building image with Conda, conda-build and
-conda-verify installed. Additional Dockerfiles exist for this image to
-allow variant to be made that support building iRODS 4.1 and iRODS 4.2.
+conda-verify installed.
 
-### irods/ubuntu/12.04 ###
+### conda/centos/7 ###
 
-This is a Docker image of a vanilla iRODS 4.1.12 server that works out
-of the box. To be used for running tests only.
+A Conda package-building image with Conda, conda-build and
+conda-verify installed.
 
 ### irods/ubuntu/16.04 ###
 
@@ -35,15 +33,34 @@ of the box. To be used for running tests only.
 
 ### irods/ubuntu/18.04 ###
 
-This is a Docker image of a vanilla iRODS 4.2.8 server that works out
-of the box. To be used for running tests only.
+This is a Docker image of a vanilla iRODS 4.2.8 / 4.2.9 server that
+works out of the box. To be used for running tests only.
+
+The server version may be chosen by passing the Docker build argument
+`--build-arg IRODS_VERSION=<version>` (default is 4.2.8).
+
+## Build instructions ##
+
+A makefile is supplied that will by default build all images and add
+metadata to them based on `git describe`.
+
+`cd ./docker`
+`make`
+
+# NPG Singularity images
+
+## The images
+
+### singularity/baton/baton-2.1.0 ###
+
+This is a Singularity image of baton 2.1.0 built for iRODS 4.2.7.
 
-## Build in instructions ##
+## Build instructions ##
 
 A makefile is supplied that will by default build all images and add
 metadata to them based on `git describe`.
 
-`cd containers`
+`cd ./singularity`
 `make`
 
 ## Author

docker/Makefile

@@ -1,26 +1,33 @@
-TAG=$(shell git describe)
+TAG=$(shell git describe --always)
 LABEL_NAMESPACE=uk.ac.sanger
-TAG_NAMESPACE=wsinpg
+ifdef REPO_OWNER
+TAG_NAMESPACE?=ghcr.io/$(REPO_OWNER)
+else
+TAG_NAMESPACE?=wsinpg
+endif
 
 DOCKER_ARGS ?= --pull --no-cache # IMPORTANT to get security updates
 
 .PHONY: clean
 
-image_names := ub-12.04-base ub-16.04-base ub-18.04-base
-image_names += ub-12.04-conda
+image_names := ub-16.04-base ub-18.04-base
 image_names += ub-16.04-irods-4.2.7
 image_names += ub-18.04-irods-4.2.8
+image_names += ub-18.04-irods-4.2.9
+
+image_names += centos-7-base
+image_names += centos-7-conda
 
 images := $(addsuffix .$(TAG), $(image_names))
 
 all: $(images)
 
-ub-12.04-base.$(TAG): base/ubuntu/12.04/Dockerfile
+centos-7-base.$(TAG): base/centos/7/Dockerfile
 	docker build $(DOCKER_ARGS) \
 	--label $(LABEL_NAMESPACE).repository=$(shell git remote get-url origin) \
 	--label $(LABEL_NAMESPACE).commit=$(shell git log --pretty=format:'%H' -n 1) \
-	--tag $(TAG_NAMESPACE)/ub-12.04-base:latest \
-	--tag $(TAG_NAMESPACE)/ub-12.04-base:$(TAG) --file $< ./base
+	--tag $(TAG_NAMESPACE)/centos-7-base:latest \
+	--tag $(TAG_NAMESPACE)/centos-7-base:$(TAG) --file $< ./base
 	touch $@
 
 ub-16.04-base.$(TAG): base/ubuntu/16.04/Dockerfile
@@ -39,12 +46,12 @@ ub-18.04-base.$(TAG): base/ubuntu/18.04/Dockerfile
 	--tag $(TAG_NAMESPACE)/ub-18.04-base:$(TAG) --file $< ./base
 	touch $@
 
-ub-12.04-conda.$(TAG): conda/ubuntu/12.04/Dockerfile ub-12.04-base.$(TAG)
+centos-7-conda.$(TAG): conda/centos/7/Dockerfile centos-7-base.$(TAG)
 	docker build $(DOCKER_ARGS) \
 	--label $(LABEL_NAMESPACE).repository=$(shell git remote get-url origin) \
 	--label $(LABEL_NAMESPACE).commit=$(shell git log --pretty=format:'%H' -n 1) \
-	--tag $(TAG_NAMESPACE)/ub-12.04-conda:latest \
-	--tag $(TAG_NAMESPACE)/ub-12.04-conda:$(TAG) --file $< ./conda
+	--tag $(TAG_NAMESPACE)/centos-7-conda:latest \
+	--tag $(TAG_NAMESPACE)/centos-7-conda:$(TAG) --file $< ./conda
 	touch $@
 
 ub-16.04-irods-4.2.7.$(TAG): irods/ubuntu/16.04/Dockerfile ub-16.04-base.$(TAG)
@@ -63,5 +70,13 @@ ub-18.04-irods-4.2.8.$(TAG): irods/ubuntu/18.04/Dockerfile ub-18.04-base.$(TAG)
 	--tag $(TAG_NAMESPACE)/ub-18.04-irods-4.2.8:$(TAG) --file $< ./irods
 	touch $@
 
+ub-18.04-irods-4.2.9.$(TAG): irods/ubuntu/18.04/Dockerfile ub-18.04-base.$(TAG)
+	docker build $(DOCKER_ARGS) --build-arg IRODS_VERSION=4.2.9 \
+	--label $(LABEL_NAMESPACE).repository=$(shell git remote get-url origin) \
+	--label $(LABEL_NAMESPACE).commit=$(shell git log --pretty=format:'%H' -n 1) \
+	--tag $(TAG_NAMESPACE)/ub-18.04-irods-4.2.9:latest \
+	--tag $(TAG_NAMESPACE)/ub-18.04-irods-4.2.9:$(TAG) --file $< ./irods
+	touch $@
+
 clean:
 	rm -f $(foreach image_name,$(image_names), $(image_name)*)

docker/base/centos/7/Dockerfile

@@ -0,0 +1,19 @@
+FROM centos:7 as installer
+
+ENV GOSU_VERSION=1.11
+
+COPY ./scripts /opt/docker/base/scripts
+
+RUN yum -y update && \
+    yum -y install curl
+
+RUN GOSU_VERSION=$GOSU_VERSION /opt/docker/base/scripts/install_gosu.sh && \
+    chmod +x /opt/docker/base/scripts/docker-entrypoint.sh
+
+FROM centos:7
+
+COPY --from=installer /usr/local/bin/gosu /usr/local/bin/gosu
+COPY --from=installer /opt/docker/base/scripts/docker-entrypoint.sh \
+    /opt/docker/base/scripts/docker-entrypoint.sh
+
+ENTRYPOINT ["/opt/docker/base/scripts/docker-entrypoint.sh"]

docker/base/centos/7/README.md

@@ -0,0 +1 @@
+A base image with gosu installed.

docker/base/scripts/docker-entrypoint.sh

@@ -4,17 +4,18 @@
 #
 # https://denibertovic.com/posts/handling-permissions-with-docker-volumes/
 #
-# Add a local using the USER and UID passed in, or fallback.
+# Add a local user using USER_NAME and USER_ID passed in, or fallback to
+# default user name "duser" and default user ID 1001.
 
-UID=${UID:-1001}
-USER=${USER:-user}
+USER_ID=${USER_ID:-1001}
+USER_NAME=${USER_NAME:-duser}
 
-echo "Starting with USER: $USER, UID: $UID"
+echo "Starting with USER: $USER_NAME, UID: $USER_ID"
 
-useradd --shell /bin/bash --uid $UID \
-        --non-unique --create-home $USER --comment ""
+useradd --shell /bin/bash --uid $USER_ID \
+        --non-unique --create-home $USER_NAME --comment ""
 
-export HOME=/home/$USER
-chown $USER:$USER $HOME
+export HOME=/home/$USER_NAME
+chown $USER_NAME:$USER_NAME $HOME
 
-exec gosu $USER "$@"
+exec gosu $USER_NAME "$@"

docker/base/ubuntu/12.04/README.md

@@ -1,2 +1,2 @@
 A base image with curl and gosu installed, intended for multi-stage
-builds where curl is later dropped.
+builds where curl is later dropped.

docker/base/ubuntu/16.04/README.md

@@ -1,2 +1,2 @@
 A base image with curl and gosu installed, intended for multi-stage
-builds where curl is later dropped.
+builds where curl is later dropped.

docker/base/ubuntu/18.04/README.md

@@ -1,2 +1,2 @@
 A base image with curl and gosu installed, intended for multi-stage
-builds where curl is later dropped.
+builds where curl is later dropped.

docker/conda/centos/7/Dockerfile

@@ -0,0 +1,47 @@
+FROM wsinpg/centos-7-base:latest as installer
+
+ENV CONDA_GROUP_ID=32001 \
+    LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8
+
+COPY . /opt/docker/conda
+
+RUN yum -y update && \
+    yum -y install \
+    bzip2 \
+    glibc-langpack-en && \
+    localedef -i en_GB -c -f UTF-8 \
+    -A /usr/share/locale/locale.alias en_GB.UTF-8 && \
+    /opt/docker/conda/scripts/install_conda.sh && \
+    cp /root/.condarc /opt/conda/etc/condarc && \
+    chmod +x /opt/docker/conda/scripts/docker-entrypoint.sh && \
+    groupadd --gid $CONDA_GROUP_ID conda && \
+    chgrp -R conda /opt/conda && \
+    chmod -R g=u /opt/conda
+
+FROM wsinpg/centos-7-base:latest
+
+ENV CONDA_GROUP_ID=32001 \
+    LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8
+
+RUN yum -y update && \
+    yum install -y \
+    glibc-langpack-en \
+    rsync \
+    vim && \
+    localedef -i en_GB -c -f UTF-8 \
+    -A /usr/share/locale/locale.alias en_GB.UTF-8 && \
+    yum autoremove -y && \
+    yum -y clean all && \
+    groupadd -g $CONDA_GROUP_ID conda
+
+COPY --from=installer --chown=0:conda /opt/conda /opt/conda
+COPY --from=installer /opt/docker/conda/scripts/docker-entrypoint.sh \
+    /opt/docker/conda/scripts/docker-entrypoint.sh
+
+ENTRYPOINT ["/opt/docker/conda/scripts/docker-entrypoint.sh"]
+
+CMD ["/bin/bash"]

docker/conda/scripts/docker-entrypoint.sh

@@ -25,8 +25,14 @@ if [ ! -e "$HOME/.condarc" ]; then
     cp "$CONDA_INSTALL_DIR/etc/condarc" "$HOME/.condarc"
 fi
 
-chown -R $USER:$USER /home/$USER
-chown -R $USER:conda "$CONDA_INSTALL_DIR"
+find /home/$USER -not -user $USER -not -group $USER -print0 |\
+    xargs --max-procs 0 --null --no-run-if-empty \
+	  chown --no-dereference $USER:$USER
+
+find "$CONDA_INSTALL_DIR" -not -user $USER -not -group conda -print0 |\
+    xargs --max-procs 0 --null --no-run-if-empty \
+	  chown --no-dereference $USER:conda
+
 chmod g+w "$CONDA_INSTALL_DIR"
 
 exec gosu $USER "$@"

docker/conda/scripts/install_conda.sh

@@ -19,13 +19,13 @@ make_temp_dir() {
 # Conda parameters
 OS=${OS:-"Linux"}
 ARCH=${ARCH:-"x86_64"}
-PYTHON_VERSION=${PYTHON_VERSION:-"3"}
-CONDA_VERSION=${CONDA_VERSION:-"4.6.14"}
-CONDA_SHA256=${CONDA_SHA256:-"0d6b23895a91294a4924bd685a3a1f48e35a17970a073cd2f684ffe2c31fc4be"}
-CONDA_URL="https://repo.anaconda.com/miniconda/Miniconda$PYTHON_VERSION-$CONDA_VERSION-$OS-$ARCH.sh"
+PYTHON_VERSION=${PYTHON_VERSION:-"39"}
+CONDA_VERSION=${CONDA_VERSION:-"4.9.2"}
+CONDA_SHA256=${CONDA_SHA256:-"536817d1b14cb1ada88900f5be51ce0a5e042bae178b5550e62f61e223deae7c"}
+CONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py${PYTHON_VERSION}_${CONDA_VERSION}-${OS}-${ARCH}.sh"
 
 # Conda package parameters
-CONDA_BUILD_VERSION=${CONDA_BUILD_VERSION:-"3.18.11"}
+CONDA_BUILD_VERSION=${CONDA_BUILD_VERSION:-"3.21.4"}
 
 CONDA_INSTALL_DIR=${CONDA_INSTALL_DIR:-/opt/conda}
 WORK_DIR=$(make_temp_dir)
@@ -42,9 +42,8 @@ conda config --set auto_update_conda False
 conda config --set ssl_verify True
 conda config --set show_channel_urls True
 
+conda install -y -n base "conda-build=$CONDA_BUILD_VERSION" conda-verify
 conda install -y jinja2 networkx packaging pyaml setuptools
-conda install -y "conda-build=$CONDA_BUILD_VERSION" conda-verify
+conda install -y curl git patch
 
-pip install rfc3987
-
-conda clean -y -st
+conda clean -y --all

docker/conda/ubuntu/12.04/Dockerfile → docker/conda/ubuntu/16.04/Dockerfile

@@ -1,4 +1,4 @@
-FROM wsinpg/ub-12.04-base:latest as installer
+FROM wsinpg/ub-16.04-base:latest as installer
 
 ENV CONDA_GROUP_ID=32001 \
     LANG=en_GB.UTF-8 \
@@ -22,7 +22,7 @@ RUN apt-get update && \
     chgrp -R conda /opt/conda && \
     chmod -R g=u /opt/conda
 
-FROM wsinpg/ub-12.04-base:latest
+FROM wsinpg/ub-16.04-base:latest
 
 ENV CONDA_GROUP_ID=32001 \
     LANG=en_GB.UTF-8 \

docker/conda/ubuntu/16.04/README.md

@@ -0,0 +1,2 @@
+A Conda package-building image with Conda, conda-build and
+conda-verify installed.

docker/irods/scripts/configure_irods.sh

@@ -33,6 +33,11 @@ cd /var/lib/irods/.irods/
 echo $(jq -f /opt/docker/irods/config/irods_environment.delta \
           ./irods_environment.json) > ./irods_environment.json
 
+# iRODS 4.2.9 needs to started here. It's not required for 4.2.7 and
+# 4.2.8, which appear to be started during setup. We call restart
+# because trying to start a server fails if one is already running.
+sudo service irods restart
+
 # Fix up the demoResc to use localhost, to avoid the transient
 # hostname of the build container being captured in the IES database.
 sudo su irods -c "iadmin modresc demoResc host localhost"