From 0afd95d7a6af71c1da05ac790addd942552b96a7 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 10:54:31 +0100
Subject: [PATCH 01/62] Workflow to build docker images

---
 .github/workflows/docker.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 .github/workflows/docker.yml

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000..8759cdb
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,26 @@
+---
+# yamllint disable rule:line-length
+name: Docker
+# yamllint disable-line rule:truthy
+on:
+    push:
+        paths:
+            - '.github/workflows/docker.yml'
+            - 'intermine_builder/**'
+            - 'local.docker-compose.yml'
+            - 'mkdatadirs.sh'
+            - 'postgres/**'
+            - 'solr/**'
+            - 'tomcat/**'
+
+jobs:
+    docker:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Build
+              run: |
+                  set -eux -o pipefail
+                  ./mkdatadirs.sh local.docker-compose.yml
+                  UID=$(id -u) GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d

From edc3cb0dc66dd791c394ffe4ac32a534ea958152 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 10:54:57 +0100
Subject: [PATCH 02/62] Ignore Emacs autosave files

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index fd80175..7a271ba 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .env
 /data
+*~

From 2ec09d4961dc5ba7d2fc74e7a50ece18ae9b450f Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 10:55:36 +0100
Subject: [PATCH 03/62] Stacktrace from gradlew cargoDeployRemote

---
 intermine_builder/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 0879302..1ae48d1 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -5,9 +5,9 @@ if [ -d ${MINE_NAME:-biotestmine} ] && [ ! -z "$(ls -A ${MINE_NAME:-biotestmine}
     echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp"
     cd /home/intermine/intermine
     cd ${MINE_NAME:-biotestmine}
-    ./gradlew cargoDeployRemote
+    ./gradlew cargoDeployRemote --stacktrace
     sleep 60
-    ./gradlew cargoRedeployRemote  --stacktrace
+    ./gradlew cargoRedeployRemote --stacktrace
     exit 0
 fi
 

From 74f67c5194205bc8dedd6346b25917729424b13d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 11:00:14 +0100
Subject: [PATCH 04/62] Avoid readonly variables in docker compose script

---
 .github/workflows/docker.yml | 2 +-
 local.docker-compose.yml     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 8759cdb..6086687 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -23,4 +23,4 @@ jobs:
               run: |
                   set -eux -o pipefail
                   ./mkdatadirs.sh local.docker-compose.yml
-                  UID=$(id -u) GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d
diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index 8e057d5..48cc6a1 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -5,7 +5,7 @@ services:
     build:
       context: ./intermine_builder
       dockerfile: ./intermine_builder.Dockerfile
-    user: ${UID:-1000}:${GID:-1000}
+    user: ${DOCKER_UID:-1000}:${DOCKER_GID:-1000}
     volumes:
       # - ./data/mine/data:/home/intermine/intermine/data
       - ./data/mine/dump:/home/intermine/intermine/dump
@@ -21,7 +21,7 @@ services:
       - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"}
       - IM_REPO_URL=${IM_REPO_URL:-}
       - IM_REPO_BRANCH=${IM_REPO_BRANCH:-}
-    depends_on: 
+    depends_on:
       - postgres
       - solr
       - tomcat
@@ -31,7 +31,7 @@ services:
     build:
       context: ./postgres
       dockerfile: ./postgres.Dockerfile
-    user: ${UID:-1000}:${GID:-1000}
+    user: ${DOCKER_UID:-1000}:${DOCKER_GID:-1000}
     volumes:
       - ./data/postgres:/var/lib/postgresql/data
 
@@ -43,7 +43,7 @@ services:
     environment:
       - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"}
       - MINE_NAME=${MINE_NAME:-biotestmine}
-    user: ${UID:-1000}:${GID:-1000}
+    user: ${DOCKER_UID:-1000}:${DOCKER_GID:-1000}
     volumes:
       - ./data/solr:/var/solr
 

From 2571098e0e2ba6ab78d78ca5e909923625e8ffed Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 11:30:22 +0100
Subject: [PATCH 05/62] Check webapp in docker workflow

---
 .github/workflows/docker.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6086687..8ad5ebc 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -22,5 +22,9 @@ jobs:
             - name: Build
               run: |
                   set -eux -o pipefail
+                  sudo apt -y install wait-for-it
                   ./mkdatadirs.sh local.docker-compose.yml
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d
+                  wait-for-it localhost:9999 --timeout=300
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs
+                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"

From af17cb73146d66dbb275d2458c174ba872fc6f78 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 11:40:26 +0100
Subject: [PATCH 06/62] Split up steps of Docker workflow

---
 .github/workflows/docker.yml | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 8ad5ebc..d757e88 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -19,12 +19,24 @@ jobs:
         steps:
             - uses: actions/checkout@v4
 
-            - name: Build
+            - name: Install prerequisites
               run: |
                   set -eux -o pipefail
                   sudo apt -y install wait-for-it
+            - name: Make data dirs
+              run: |
+                  set -eux -o pipefail
                   ./mkdatadirs.sh local.docker-compose.yml
+            - name: Build
+              run: |
+                  set -eux -o pipefail
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d
                   wait-for-it localhost:9999 --timeout=300
+            - name: Dump Docker logs
+              run: |
+                  set -eux -o pipefail
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs
+            - name: Check webapp
+              run: |
+                  set -eux -o pipefail
                   curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"

From bebb6aaced9a716c73bbfbf7dca7b58ed3749d56 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 11:53:44 +0100
Subject: [PATCH 07/62] Try waiting a bit before dumping logs

They are getting truncated
---
 .github/workflows/docker.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index d757e88..7b9d70f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -35,6 +35,7 @@ jobs:
             - name: Dump Docker logs
               run: |
                   set -eux -o pipefail
+                  sleep 60
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs
             - name: Check webapp
               run: |

From 21ec0cd3e9c068cf9d4b7f2f1a8366d4de73c58b Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 11 Jul 2024 12:06:26 +0100
Subject: [PATCH 08/62] Try to get a bit more log output

---
 .github/workflows/docker.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 7b9d70f..b568771 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -35,8 +35,11 @@ jobs:
             - name: Dump Docker logs
               run: |
                   set -eux -o pipefail
-                  sleep 60
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs
+                  sleep 120
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs solr
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs tomcat
             - name: Check webapp
               run: |
                   set -eux -o pipefail

From a480c039683d38682436e295fc1806da0ff8a86d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:13:14 +0100
Subject: [PATCH 09/62] Increase workflow timeout to 15 minutes

---
 .github/workflows/docker.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b568771..7bb1e56 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -31,7 +31,8 @@ jobs:
               run: |
                   set -eux -o pipefail
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d
-                  wait-for-it localhost:9999 --timeout=300
+                  # Wait 15 minutes
+                  wait-for-it localhost:9999 --timeout=900
             - name: Dump Docker logs
               run: |
                   set -eux -o pipefail

From 97a1f9a807edcafd7337e4b19c5b9f4f16a8e0ed Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:17:03 +0100
Subject: [PATCH 10/62] Use camCHILDMine in workflow

---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 7bb1e56..549cc18 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -30,7 +30,7 @@ jobs:
             - name: Build
               run: |
                   set -eux -o pipefail
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up --build -d
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up --build -d
                   # Wait 15 minutes
                   wait-for-it localhost:9999 --timeout=900
             - name: Dump Docker logs

From f0355c0f0f48a84ebb12a5a0b742fe344b955c73 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:22:24 +0100
Subject: [PATCH 11/62] Refactor build script mine name variable

---
 intermine_builder/build.sh | 102 +++++++++++++++++++------------------
 1 file changed, 52 insertions(+), 50 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 1ae48d1..eb606c7 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -1,10 +1,12 @@
 #!/bin/bash
 
-if [ -d ${MINE_NAME:-biotestmine} ] && [ ! -z "$(ls -A ${MINE_NAME:-biotestmine})" ] && [ ! $FORCE_MINE_BUILD ]; then
-    echo "$(date +%Y/%m/%d-%H:%M) Mine already exists"
+THE_MINE_NAME=${MINE_NAME:-biotestmine}
+
+if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then
+    echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists"
     echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp"
     cd /home/intermine/intermine
-    cd ${MINE_NAME:-biotestmine}
+    cd ${THE_MINE_NAME}
     ./gradlew cargoDeployRemote --stacktrace
     sleep 60
     ./gradlew cargoRedeployRemote --stacktrace
@@ -44,71 +46,71 @@ fi
 echo "Starting mine build"
 echo $MINE_REPO_URL
 # Check if mine exists and is not empty
-if [ -d ${MINE_NAME:-biotestmine} ] && [ ! -z "$(ls -A ${MINE_NAME:-biotestmine})" ]; then
-    echo "$(date +%Y/%m/%d-%H:%M) Update ${MINE_NAME:-biotestmine} to newest version" #>> /home/intermine/intermine/build.progress
-    cd ${MINE_NAME:-biotestmine}
+if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ]; then
+    echo "$(date +%Y/%m/%d-%H:%M) Update ${THE_MINE_NAME} to newest version" #>> /home/intermine/intermine/build.progress
+    cd ${THE_MINE_NAME}
     # git pull
     cd /home/intermine/intermine
 else
-    # echo "$(date +%Y/%m/%d-%H:%M) Clone ${MINE_NAME:-biotestmine}" #>> /home/intermine/intermine/build.progress
-    echo "$(date +%Y/%m/%d-%H:%M) Clone ${MINE_NAME:-biotestmine}"
-    git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${MINE_NAME:-biotestmine}
+    # echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}" #>> /home/intermine/intermine/build.progress
+    echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}"
+    git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${THE_MINE_NAME}
     echo "$(date +%Y/%m/%d-%H:%M) Update keyword_search.properties to use http://solr" #>> /home/intermine/intermine/build.progress
-    sed -i 's/localhost/'${SOLR_HOST:-solr}'/g' ./${MINE_NAME:-biotestmine}/dbmodel/resources/keyword_search.properties
+    sed -i 's/localhost/'${SOLR_HOST:-solr}'/g' ./${THE_MINE_NAME}/dbmodel/resources/keyword_search.properties
 fi
 
 # If InterMine or Bio versions have been set (likely because of a custom
 # InterMine build), update gradle.properties in the mine.
 if [ ! -z ${IM_VERSION} ]; then
-    sed -i "s/\(systemProp\.imVersion=\).*\$/\1${IM_VERSION}/" /home/intermine/intermine/${MINE_NAME:-biotestmine}/gradle.properties
+    sed -i "s/\(systemProp\.imVersion=\).*\$/\1${IM_VERSION}/" /home/intermine/intermine/${THE_MINE_NAME}/gradle.properties
 fi
 if [ ! -z ${BIO_VERSION} ]; then
-    sed -i "s/\(systemProp\.bioVersion=\).*\$/\1${BIO_VERSION}/" /home/intermine/intermine/${MINE_NAME:-biotestmine}/gradle.properties
+    sed -i "s/\(systemProp\.bioVersion=\).*\$/\1${BIO_VERSION}/" /home/intermine/intermine/${THE_MINE_NAME}/gradle.properties
 fi
 
 # Copy project_build from intermine_scripts repo
-if [ ! -f /home/intermine/intermine/${MINE_NAME:-biotestmine}/project_build ]; then
+if [ ! -f /home/intermine/intermine/${THE_MINE_NAME}/project_build ]; then
     echo "$(date +%Y/%m/%d-%H:%M) Cloning intermine scripts repo to /home/intermine/intermine/intermine-scripts"
     git clone https://github.com/intermine/intermine-scripts
-    echo "$(date +%Y/%m/%d-%H:%M) Copy project_build to /home/intermine/intermine/${MINE_NAME:-biotestmine}"
-    cp /home/intermine/intermine/intermine-scripts/project_build /home/intermine/intermine/${MINE_NAME:-biotestmine}/project_build
-    chmod +x /home/intermine/intermine/${MINE_NAME:-biotestmine}/project_build
+    echo "$(date +%Y/%m/%d-%H:%M) Copy project_build to /home/intermine/intermine/${THE_MINE_NAME}"
+    cp /home/intermine/intermine/intermine-scripts/project_build /home/intermine/intermine/${THE_MINE_NAME}/project_build
+    chmod +x /home/intermine/intermine/${THE_MINE_NAME}/project_build
 fi
 
 # Copy mine properties
-if [ ! -f /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties ]; then
-    if [ ! -f /home/intermine/intermine/configs/${MINE_NAME:-biotestmine}.properties ]; then
-        echo "$(date +%Y/%m/%d-%H:%M) Copy biotestmine.properties to ~/.intermine/${MINE_NAME:-biotestmine}.properties" #>> /home/intermine/intermine/build.progress
-        cp /home/intermine/intermine/${MINE_NAME:-biotestmine}/data/${MINE_NAME:-biotestmine}.properties /home/intermine/.intermine/
+if [ ! -f /home/intermine/.intermine/${THE_MINE_NAME}.properties ]; then
+    if [ ! -f /home/intermine/intermine/configs/${THE_MINE_NAME}.properties ]; then
+        echo "$(date +%Y/%m/%d-%H:%M) Copy biotestmine.properties to ~/.intermine/${THE_MINE_NAME}.properties" #>> /home/intermine/intermine/build.progress
+        cp /home/intermine/intermine/${THE_MINE_NAME}/data/${THE_MINE_NAME}.properties /home/intermine/.intermine/
     else
-        echo "$(date +%Y/%m/%d-%H:%M) Copy ${MINE_NAME:-biotestmine}.properties to ~/.intermine/${MINE_NAME:-biotestmine}.properties"
-        cp /home/intermine/intermine/configs/${MINE_NAME:-biotestmine}.properties /home/intermine/.intermine/
+        echo "$(date +%Y/%m/%d-%H:%M) Copy ${THE_MINE_NAME}.properties to ~/.intermine/${THE_MINE_NAME}.properties"
+        cp /home/intermine/intermine/configs/${THE_MINE_NAME}.properties /home/intermine/.intermine/
     fi
 
-    echo -e "$(date +%Y/%m/%d-%H:%M) Set properties in .intermine/${MINE_NAME:-biotestmine}.properties to\nPSQL_DB_NAME\tbiotestmine\nPSQL_USER\t$PSQL_USER\nPSQL_PWD\t$PSQL_PWD\nTOMCAT_USER\t$TOMCAT_USER\nTOMCAT_PWD\t$TOMCAT_PWD\nGRADLE_OPTS\t$GRADLE_OPTS" #>> /home/intermine/intermine/build.progress
+    echo -e "$(date +%Y/%m/%d-%H:%M) Set properties in .intermine/${THE_MINE_NAME}.properties to\nPSQL_DB_NAME\tbiotestmine\nPSQL_USER\t$PSQL_USER\nPSQL_PWD\t$PSQL_PWD\nTOMCAT_USER\t$TOMCAT_USER\nTOMCAT_PWD\t$TOMCAT_PWD\nGRADLE_OPTS\t$GRADLE_OPTS" #>> /home/intermine/intermine/build.progress
 
-    #sed -i "s/PSQL_PORT/$PGPORT/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/PSQL_DB_NAME/${MINE_NAME:-biotestmine}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/PSQL_USER/${PSQL_USER:-postgres}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/PSQL_PWD/${PSQL_PWD:-postgres}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${TOMCAT_HOST:-tomcat}:${TOMCAT_PORT:-8080}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    sed -i "s/serverName=localhost/serverName=${PGHOST:-postgres}:${PGPORT:-5432}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
+    #sed -i "s/PSQL_PORT/$PGPORT/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/PSQL_DB_NAME/${THE_MINE_NAME}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/PSQL_USER/${PSQL_USER:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/PSQL_PWD/${PSQL_PWD:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${TOMCAT_HOST:-tomcat}:${TOMCAT_PORT:-8080}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/serverName=localhost/serverName=${PGHOST:-postgres}:${PGPORT:-5432}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
 
 
-    # echo "project.rss=http://localhost:$WORDPRESS_PORT/?feed=rss2" >> /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
-    # echo "links.blog=https://localhost:$WORDPRESS_PORT" >> /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties
+    # echo "project.rss=http://localhost:$WORDPRESS_PORT/?feed=rss2" >> /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    # echo "links.blog=https://localhost:$WORDPRESS_PORT" >> /home/intermine/.intermine/${THE_MINE_NAME}.properties
 fi
 
 # Copy mine configs
-if [ ! -f /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml ]; then
+if [ ! -f /home/intermine/intermine/${THE_MINE_NAME}/project.xml ]; then
     if [ -f /home/intermine/intermine/configs/project.xml ]; then
-        echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/${MINE_NAME:-biotestmine}/project.xml"
-        cp /home/intermine/intermine/configs/project.xml /home/intermine/intermine/${MINE_NAME:-biotestmine}/
+        echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/${THE_MINE_NAME}/project.xml"
+        cp /home/intermine/intermine/configs/project.xml /home/intermine/intermine/${THE_MINE_NAME}/
         echo "$(date +%Y/%m/%d-%H:%M) Set correct source path in project.xml"
-        sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml
-        sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml
+        sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
+        sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
     else
         echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/biotestmine/project.xml" #>> /home/intermine/intermine/build.progress
         cp /home/intermine/intermine/biotestmine/data/project.xml /home/intermine/intermine/biotestmine/
@@ -120,8 +122,8 @@ if [ ! -f /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml ]; the
     fi
 else
     echo "$(date +%Y/%m/%d-%H:%M) Set correct source path in project.xml"
-    sed -i "s~${IM_DATA_DIR:-DATA_DIR}~/home/intermine/intermine/data~g" /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml
-    sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml
+    sed -i "s~${IM_DATA_DIR:-DATA_DIR}~/home/intermine/intermine/data~g" /home/intermine/intermine/${THE_MINE_NAME}/project.xml
+    sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
 fi
 
 # Copy data
@@ -164,9 +166,9 @@ echo "$(date +%Y/%m/%d-%H:%M) Database is now available ..." #>> /home/intermine
 echo "$(date +%Y/%m/%d-%H:%M) Reset databases and roles" #>> /home/intermine/intermine/build.progress
 
 # Delete Databases if exist
-psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"${MINE_NAME:-biotestmine}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"items-${MINE_NAME:-biotestmine}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"userprofile-${MINE_NAME:-biotestmine}\";"
+psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"${THE_MINE_NAME}\";"
+psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"items-${THE_MINE_NAME}\";"
+psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"userprofile-${THE_MINE_NAME}\";"
 
 # psql -U postgres -h ${PGHOST:-postgres} -c "DROP ROLE IF EXISTS ${PSQL_USER:-postgres};"
 
@@ -174,15 +176,15 @@ psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"userprofil
 echo "$(date +%Y/%m/%d-%H:%M) Creating postgres database tables and roles.." #>> /home/intermine/intermine/build.progress
 # psql -U postgres -h ${PGHOST:-postgres} -c "CREATE USER ${PSQL_USER:-postgres} WITH PASSWORD '${PSQL_PWD:-postgres}';"
 psql -U postgres -h ${PGHOST:-postgres} -c "ALTER USER ${PSQL_USER:-postgres} WITH SUPERUSER;"
-psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"${MINE_NAME:-biotestmine}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"items-${MINE_NAME:-biotestmine}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"userprofile-${MINE_NAME:-biotestmine}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE ${MINE_NAME:-biotestmine} to ${PSQL_USER:-postgres};"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${MINE_NAME:-biotestmine}\" to ${PSQL_USER:-postgres};"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${MINE_NAME:-biotestmine}\" to ${PSQL_USER:-postgres};"
+psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"${THE_MINE_NAME}\";"
+psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"items-${THE_MINE_NAME}\";"
+psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"userprofile-${THE_MINE_NAME}\";"
+psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE ${THE_MINE_NAME} to ${PSQL_USER:-postgres};"
+psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
+psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
 
 
-cd ${MINE_NAME:-biotestmine}
+cd ${THE_MINE_NAME}
 
 echo "$(date +%Y/%m/%d-%H:%M) Running project_build script"
 ./project_build -b -T localhost /home/intermine/intermine/dump/dump

From fe667dbb25fd3821fcfb7e1335f71f89d1ecfdec Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:23:02 +0100
Subject: [PATCH 12/62] camCHILDMine requires bash in the builder Docker image

---
 intermine_builder/intermine_builder.Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile
index 4dab998..2aed1de 100644
--- a/intermine_builder/intermine_builder.Dockerfile
+++ b/intermine_builder/intermine_builder.Dockerfile
@@ -6,7 +6,8 @@ ENV JAVA_HOME="/usr/lib/jvm/default-jvm"
 RUN apk add --no-cache openjdk8 openjdk8-jre && \
     ln -sf "${JAVA_HOME}/bin/"* "/usr/bin/"
 
-RUN apk add --no-cache git \
+RUN apk add --no-cache bash \
+                       git \
                        maven \
                        postgresql-client \
                        perl \

From bbfa848fc08006b6685150d9514c9e39734d009e Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:26:00 +0100
Subject: [PATCH 13/62] Allow mine data volume to be overridden

---
 local.docker-compose.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index 48cc6a1..07113f9 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -12,8 +12,7 @@ services:
       - ./data/mine/configs:/home/intermine/intermine/configs
       - ./data/mine/packages:/home/intermine/.m2
       - ./data/mine/intermine:/home/intermine/.intermine
-      # - ./data/mine/[PUT_YOUR_MINE_NAME_HERE]:/home/intermine/intermine/[PUT_YOUR_MINE_NAME_HERE]
-      - ./data/mine/biotestmine:/home/intermine/intermine/biotestmine
+      - ./data/mine/${MINE_NAME:-biotestmine}:/home/intermine/intermine/${MINE_NAME:-biotestmine}
     environment:
       - MINE_NAME=${MINE_NAME:-biotestmine}
       - MINE_REPO_URL=${MINE_REPO_URL:-}

From 532562f000d9291a165a324f0b63fe4a03e9bf05 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:34:49 +0100
Subject: [PATCH 14/62] Check the right webapp in the Docker workflow

---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 549cc18..6f11a78 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -44,4 +44,4 @@ jobs:
             - name: Check webapp
               run: |
                   set -eux -o pipefail
-                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"
+                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/camCHILDMine/"

From 20ea19d84b02e78a9f6c7d88f7ec590906b54684 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:35:07 +0100
Subject: [PATCH 15/62] Abort on build script failure

---
 intermine_builder/build.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index eb606c7..4294503 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -eux -o pipefail
+
 THE_MINE_NAME=${MINE_NAME:-biotestmine}
 
 if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then

From 76597f72b613b543f410c2f7b4278a02b0fa837b Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:57:00 +0100
Subject: [PATCH 16/62] clone camCHILDMine in workflow

---
 .github/workflows/docker.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6f11a78..56bc94d 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -27,9 +27,15 @@ jobs:
               run: |
                   set -eux -o pipefail
                   ./mkdatadirs.sh local.docker-compose.yml
+            - name: Clone camCHILDMine
+              run: |
+                  set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}/data/mine
+                  git clone https://github.com/ucam-department-of-psychiatry/camCHILDMine
             - name: Build
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up --build -d
                   # Wait 15 minutes
                   wait-for-it localhost:9999 --timeout=900

From 6acd919e372c6d8acb6e4d29026c5070fad564c4 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 10:57:12 +0100
Subject: [PATCH 17/62] Remove duplicate set -e

---
 intermine_builder/build.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 4294503..8ecc478 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -15,8 +15,6 @@ if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE
     exit 0
 fi
 
-set -e
-
 cd /home/intermine/intermine
 
 # Empty log

From 57b0ff733faf85fb17924f182750ba40a3c1c310 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 11:21:23 +0100
Subject: [PATCH 18/62] Try to abort workflow early if build fails

---
 .github/workflows/docker.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 56bc94d..a1ff856 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -36,8 +36,11 @@ jobs:
               run: |
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up --build -d
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml build
+                  # Wait for builder to finish and return exit code
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up -d
                   wait-for-it localhost:9999 --timeout=900
             - name: Dump Docker logs
               run: |

From e556605351497effea6d5816b4cd15bd7f42e08a Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 11:28:54 +0100
Subject: [PATCH 19/62] Remove obsolete docker compose versions

---
 dockerhub.docker-compose.yml | 3 +--
 local.docker-compose.yml     | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/dockerhub.docker-compose.yml b/dockerhub.docker-compose.yml
index b6a2929..9070e40 100644
--- a/dockerhub.docker-compose.yml
+++ b/dockerhub.docker-compose.yml
@@ -1,4 +1,3 @@
-version: "3"
 services:
   intermine_builder:
     container_name: intermine_builder
@@ -19,7 +18,7 @@ services:
       - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"}
       - IM_REPO_URL=${IM_REPO_URL:-}
       - IM_REPO_BRANCH=${IM_REPO_BRANCH:-}
-    depends_on: 
+    depends_on:
       - postgres
       - solr
       - tomcat
diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index 07113f9..b6cd5e5 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -1,4 +1,3 @@
-version: "3"
 services:
   intermine_builder:
     container_name: intermine_builder

From cf35bcb548f9ecb358dbf27eece642e2e824f05e Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 11:29:25 +0100
Subject: [PATCH 20/62] Set default for FORCE_MINE_BUILD

---
 intermine_builder/build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 8ecc478..907a776 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -3,6 +3,7 @@
 set -eux -o pipefail
 
 THE_MINE_NAME=${MINE_NAME:-biotestmine}
+FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0}
 
 if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then
     echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists"

From 4ea2b2154b097729d18b622318a58a5e7d548800 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 11:41:21 +0100
Subject: [PATCH 21/62] Try docker compose run to get build errors from the
 builder itself

---
 .github/workflows/docker.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index a1ff856..d28c955 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -36,7 +36,8 @@ jobs:
               run: |
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml build
+                  # Try to trap any build errors early
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run intermine_builder -d
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes

From dc934cde91afc65e35222fbb8baad475cd6d38b0 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 11:41:41 +0100
Subject: [PATCH 22/62] Ignore Silver Searcher .ignore file

---
 .gitignore | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitignore b/.gitignore
index 7a271ba..bbbec6b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,8 @@
 .env
 /data
+
+# Emacs backup
 *~
+
+# Silver searcher
+.ignore

From d41faad8dd7a3fd51f2acdd6d65437e91bb12636 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 11:49:56 +0100
Subject: [PATCH 23/62] Maybe I got those arguments wrong. Also remove the
 container

---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index d28c955..16c4da5 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -37,7 +37,7 @@ jobs:
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}
                   # Try to trap any build errors early
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run intermine_builder -d
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run -d --rm intermine_builder
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes

From aa6128e7e31c8c923e209bea679a7eb6ba77f163 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 12:04:33 +0100
Subject: [PATCH 24/62] Maybe we don't need detach mode

---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 16c4da5..3e09ddd 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -37,7 +37,7 @@ jobs:
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}
                   # Try to trap any build errors early
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run -d --rm intermine_builder
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run --rm intermine_builder
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes

From 80bddbe0bf5b1ebb6430b43cea987ac2a679551d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 12:14:11 +0100
Subject: [PATCH 25/62] More uninitialised variables in the build script

---
 intermine_builder/build.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 907a776..ec6657a 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -4,6 +4,8 @@ set -eux -o pipefail
 
 THE_MINE_NAME=${MINE_NAME:-biotestmine}
 FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0}
+IM_VERSION=${IM_VERSION:-}
+BIO_VERSION=${BIO_VERSION:-}
 
 if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then
     echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists"

From ac05e96850eef8113b4baeb0349203d4fd960c3e Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 14:29:42 +0100
Subject: [PATCH 26/62] Checkout cadre-dev branch of camCHILDMine in workflow

---
 .github/workflows/docker.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 3e09ddd..ea8798f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -32,6 +32,7 @@ jobs:
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}/data/mine
                   git clone https://github.com/ucam-department-of-psychiatry/camCHILDMine
+                  git checkout cadre-dev
             - name: Build
               run: |
                   set -eux -o pipefail

From e7e940927640a678c46e98a5026e4476e091fd44 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 14:41:02 +0100
Subject: [PATCH 27/62] Need to change directory before switching git branch

---
 .github/workflows/docker.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index ea8798f..d50996f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -32,6 +32,7 @@ jobs:
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}/data/mine
                   git clone https://github.com/ucam-department-of-psychiatry/camCHILDMine
+                  cd camCHILDMine
                   git checkout cadre-dev
             - name: Build
               run: |

From dffdba793c807cda73187a31b9758803a1204551 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 14:41:36 +0100
Subject: [PATCH 28/62] Fix message in build script

---
 intermine_builder/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index ec6657a..3c1750b 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -83,7 +83,7 @@ fi
 # Copy mine properties
 if [ ! -f /home/intermine/.intermine/${THE_MINE_NAME}.properties ]; then
     if [ ! -f /home/intermine/intermine/configs/${THE_MINE_NAME}.properties ]; then
-        echo "$(date +%Y/%m/%d-%H:%M) Copy biotestmine.properties to ~/.intermine/${THE_MINE_NAME}.properties" #>> /home/intermine/intermine/build.progress
+        echo "$(date +%Y/%m/%d-%H:%M) Copy ${THE_MINE_NAME}.properties to ~/.intermine/${THE_MINE_NAME}.properties" #>> /home/intermine/intermine/build.progress
         cp /home/intermine/intermine/${THE_MINE_NAME}/data/${THE_MINE_NAME}.properties /home/intermine/.intermine/
     else
         echo "$(date +%Y/%m/%d-%H:%M) Copy ${THE_MINE_NAME}.properties to ~/.intermine/${THE_MINE_NAME}.properties"

From 3e3a40dc0ad0561c675006441dca619fa55fcc92 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 15:01:08 +0100
Subject: [PATCH 29/62] Fix up project.xml path

---
 intermine_builder/build.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 3c1750b..f318c65 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -115,12 +115,12 @@ if [ ! -f /home/intermine/intermine/${THE_MINE_NAME}/project.xml ]; then
         sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
         sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
     else
-        echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/biotestmine/project.xml" #>> /home/intermine/intermine/build.progress
-        cp /home/intermine/intermine/biotestmine/data/project.xml /home/intermine/intermine/biotestmine/
+        echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/intermine/${THE_MINE_NAME}/project.xml" #>> /home/intermine/intermine/build.progress
+        cp /home/intermine/intermine/${THE_MINE_NAME}/data/project.xml /home/intermine/intermine/${THE_MINE_NAME}
 
         echo "$(date +%Y/%m/%d-%H:%M) Set correct source path in project.xml" #>> /home/intermine/intermine/build.progress
-        sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/biotestmine/project.xml
-        sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/biotestmine/project.xml
+        sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
+        sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml
 
     fi
 else

From e003498ac6289fc58b2fc410e5634b2234e6ab18 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 16:10:47 +0100
Subject: [PATCH 30/62] Fix path to malaria-data.tar.gz in build script

Not the permanent solution. This should be ignored if biotestmine is not being used
---
 intermine_builder/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index f318c65..d471e29 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -141,9 +141,9 @@ if [ -d /home/intermine/intermine/data ]; then
 else
     echo "$(date +%Y/%m/%d-%H:%M) No user data directory found"
     if [ ! -d /home/intermine/intermine/data/malaria ]; then
-        echo "$(date +%Y/%m/%d-%H:%M) Copy malria-data to ~/data" #>> /home/intermine/intermine/build.progress
+        echo "$(date +%Y/%m/%d-%H:%M) Copy malaria-data to ~/data" #>> /home/intermine/intermine/build.progress
         mkdir -p /home/intermine/intermine/data/
-        cp /home/intermine/intermine/biotestmine/data/malaria-data.tar.gz /home/intermine/intermine/data/
+        cp /home/intermine/intermine/${THE_MINE_NAME}/data/malaria-data.tar.gz /home/intermine/intermine/data/
         cd /home/intermine/intermine/data/
         tar -xf malaria-data.tar.gz
         rm malaria-data.tar.gz

From ede97c8530a2c7d4b7dc30a48f4d4d3294ad78e1 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Mon, 22 Jul 2024 16:20:54 +0100
Subject: [PATCH 31/62] Fix problem with database name case-sensitivity

---
 intermine_builder/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index d471e29..7134a1f 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -182,7 +182,7 @@ psql -U postgres -h ${PGHOST:-postgres} -c "ALTER USER ${PSQL_USER:-postgres} WI
 psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"${THE_MINE_NAME}\";"
 psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"items-${THE_MINE_NAME}\";"
 psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"userprofile-${THE_MINE_NAME}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE ${THE_MINE_NAME} to ${PSQL_USER:-postgres};"
+psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
 psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
 psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
 

From f3b76db21cd908fc530a186d21bd7075dfdf0c8a Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Tue, 23 Jul 2024 16:38:45 +0100
Subject: [PATCH 32/62] Shuffle docker compose commands in workflow

Try the 'wait' after 'up'. With --rm it probably didn't make much sense waiting after the 'run'
---
 .github/workflows/docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index d50996f..814f48b 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -40,10 +40,10 @@ jobs:
                   cd ${GITHUB_WORKSPACE}
                   # Try to trap any build errors early
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run --rm intermine_builder
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up -d
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up -d
                   wait-for-it localhost:9999 --timeout=900
             - name: Dump Docker logs
               run: |

From 0814188824ec2abbc17b19c01148ffb52e95b93b Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Tue, 23 Jul 2024 21:41:34 +0100
Subject: [PATCH 33/62] Dump Docker logs on failure

---
 .github/workflows/docker.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 814f48b..2d340b1 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -34,18 +34,26 @@ jobs:
                   git clone https://github.com/ucam-department-of-psychiatry/camCHILDMine
                   cd camCHILDMine
                   git checkout cadre-dev
-            - name: Build
+            - name: Run
               run: |
                   set -eux -o pipefail
                   cd ${GITHUB_WORKSPACE}
                   # Try to trap any build errors early
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run --rm intermine_builder
+            - name: Up
+              run: |
+                  set -eux -o pipefail
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up -d
+            - name: Wait
+              run: |
+                  set -eux -o pipefail
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes
                   wait-for-it localhost:9999 --timeout=900
+
             - name: Dump Docker logs
+              if: failure()
               run: |
                   set -eux -o pipefail
                   sleep 120

From d066da65608f4f2c7e3d0def010714c91b49316a Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 24 Jul 2024 09:43:17 +0100
Subject: [PATCH 34/62] Enable stacktrace on cargoDeployRemote

---
 intermine_builder/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 7134a1f..312afa2 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -218,6 +218,6 @@ echo "$(date +%Y/%m/%d-%H:%M) Gradle: build userDB" #>> /home/intermine/intermin
 
 echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp" #>> /home/intermine/intermine/build.progress
 # ./gradlew clean
-./gradlew cargoDeployRemote
+./gradlew cargoDeployRemote  --stacktrace
 sleep 60
 ./gradlew cargoRedeployRemote  --stacktrace #>> /home/intermine/intermine/build.progress

From 9461268ba2eb1ff1a3cf3a8f386f10e54659ebbc Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 24 Jul 2024 15:21:43 +0100
Subject: [PATCH 35/62] Better fix for
 https://github.com/intermine/intermine/issues/2162

Use workaround https://github.com/intermine/intermine/issues/2162#issuecomment-952099300
---
 intermine_builder/build.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 312afa2..cf14248 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -13,8 +13,6 @@ if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE
     cd /home/intermine/intermine
     cd ${THE_MINE_NAME}
     ./gradlew cargoDeployRemote --stacktrace
-    sleep 60
-    ./gradlew cargoRedeployRemote --stacktrace
     exit 0
 fi
 
@@ -218,6 +216,5 @@ echo "$(date +%Y/%m/%d-%H:%M) Gradle: build userDB" #>> /home/intermine/intermin
 
 echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp" #>> /home/intermine/intermine/build.progress
 # ./gradlew clean
+# --stacktrace --debug --info --scan
 ./gradlew cargoDeployRemote  --stacktrace
-sleep 60
-./gradlew cargoRedeployRemote  --stacktrace #>> /home/intermine/intermine/build.progress

From 59c5b5eeea2ce5374f206f68b874a3996c1bdbbc Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 25 Jul 2024 10:07:52 +0100
Subject: [PATCH 36/62] Separate workflow to build biotestmine

---
 .github/workflows/biotestmine.yml             | 60 +++++++++++++++++++
 .../{docker.yml => camchildmine.yml}          |  6 +-
 2 files changed, 63 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/biotestmine.yml
 rename .github/workflows/{docker.yml => camchildmine.yml} (96%)

diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml
new file mode 100644
index 0000000..4002c17
--- /dev/null
+++ b/.github/workflows/biotestmine.yml
@@ -0,0 +1,60 @@
+---
+# yamllint disable rule:line-length
+name: Biotestmine
+# yamllint disable-line rule:truthy
+on:
+    push:
+        paths:
+            - '.github/workflows/biotestmine.yml'
+            - 'intermine_builder/**'
+            - 'local.docker-compose.yml'
+            - 'mkdatadirs.sh'
+            - 'postgres/**'
+            - 'solr/**'
+            - 'tomcat/**'
+
+jobs:
+    biotestmine:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Install prerequisites
+              run: |
+                  set -eux -o pipefail
+                  sudo apt -y install wait-for-it
+            - name: Make data dirs
+              run: |
+                  set -eux -o pipefail
+                  ./mkdatadirs.sh local.docker-compose.yml
+            - name: Run
+              run: |
+                  set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
+                  # Try to trap any build errors early
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml run --rm intermine_builder
+            - name: Up
+              run: |
+                  set -eux -o pipefail
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up -d
+            - name: Wait
+              run: |
+                  set -eux -o pipefail
+                  # Wait for builder to finish and return exit code
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml wait intermine_builder
+                  # Wait 15 minutes
+                  wait-for-it localhost:9999 --timeout=900
+
+            - name: Dump Docker logs
+              if: failure()
+              run: |
+                  set -eux -o pipefail
+                  sleep 120
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs solr
+                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs tomcat
+            - name: Check webapp
+              run: |
+                  set -eux -o pipefail
+                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"
diff --git a/.github/workflows/docker.yml b/.github/workflows/camchildmine.yml
similarity index 96%
rename from .github/workflows/docker.yml
rename to .github/workflows/camchildmine.yml
index 2d340b1..4060069 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/camchildmine.yml
@@ -1,11 +1,11 @@
 ---
 # yamllint disable rule:line-length
-name: Docker
+name: CamCHILDMine
 # yamllint disable-line rule:truthy
 on:
     push:
         paths:
-            - '.github/workflows/docker.yml'
+            - '.github/workflows/camchildmine.yml'
             - 'intermine_builder/**'
             - 'local.docker-compose.yml'
             - 'mkdatadirs.sh'
@@ -14,7 +14,7 @@ on:
             - 'tomcat/**'
 
 jobs:
-    docker:
+    camchildmine:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v4

From d1fa47e33e1c07814bb384ff87f1a7dfe07a089a Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 25 Jul 2024 16:32:45 +0100
Subject: [PATCH 37/62] Try to sort out all the permissions issues

---
 .github/workflows/biotestmine.yml             |  3 +-
 intermine_builder/build.sh                    |  6 ++
 .../intermine_builder.Dockerfile              | 23 ++++-
 local.docker-compose.yml                      | 99 ++++++++++++++-----
 mkdatadirs.sh                                 | 37 ++-----
 5 files changed, 106 insertions(+), 62 deletions(-)

diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml
index 4002c17..f7eac98 100644
--- a/.github/workflows/biotestmine.yml
+++ b/.github/workflows/biotestmine.yml
@@ -26,7 +26,8 @@ jobs:
             - name: Make data dirs
               run: |
                   set -eux -o pipefail
-                  ./mkdatadirs.sh local.docker-compose.yml
+                  ./mkdatadirs.sh
+                  mkdir ./data/mine/biotestmine
             - name: Run
               run: |
                   set -eux -o pipefail
diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index cf14248..d8b0038 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -55,6 +55,9 @@ if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ]; then
 else
     # echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}" #>> /home/intermine/intermine/build.progress
     echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}"
+    echo "User: $(id -u)"
+    echo "Group: $(id -g)"
+    ls -l
     git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${THE_MINE_NAME}
     echo "$(date +%Y/%m/%d-%H:%M) Update keyword_search.properties to use http://solr" #>> /home/intermine/intermine/build.progress
     sed -i 's/localhost/'${SOLR_HOST:-solr}'/g' ./${THE_MINE_NAME}/dbmodel/resources/keyword_search.properties
@@ -218,3 +221,6 @@ echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp" #>> /home/intermine/intermin
 # ./gradlew clean
 # --stacktrace --debug --info --scan
 ./gradlew cargoDeployRemote  --stacktrace
+
+# Debug: Keep the container going
+# tail -f /dev/null
diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile
index 2aed1de..1288497 100644
--- a/intermine_builder/intermine_builder.Dockerfile
+++ b/intermine_builder/intermine_builder.Dockerfile
@@ -1,6 +1,17 @@
 FROM alpine:3.12.5
 LABEL maintainer="Ank"
 
+# -----------------------------------------------------------------------------
+# Permissions
+# -----------------------------------------------------------------------------
+# https://vsupalov.com/docker-shared-permissions/
+
+ARG USER_ID
+ARG GROUP_ID
+
+RUN addgroup --gid $GROUP_ID intermine
+RUN adduser -D -g '' -u $USER_ID -G intermine intermine
+
 ENV JAVA_HOME="/usr/lib/jvm/default-jvm"
 
 RUN apk add --no-cache openjdk8 openjdk8-jre && \
@@ -59,9 +70,6 @@ RUN cpanm --force Ouch \
                 #  Digest::MD5 \
                 #  Log::Handler
 
-RUN mkdir /home/intermine && mkdir /home/intermine/intermine
-RUN chmod -R 777 /home/intermine
-
 ENV MEM_OPTS="-Xmx1g -Xms500m"
 ENV GRADLE_OPTS="-server ${MEM_OPTS} -XX:+UseParallelGC -XX:SoftRefLRUPolicyMSPerMB=1 -XX:MaxHeapFreeRatio=99 -Dorg.gradle.daemon=false -Duser.home=/home/intermine"
 ENV HOME="/home/intermine"
@@ -74,8 +82,15 @@ ENV TOMCAT_PWD="tomcat"
 ENV TOMCAT_PORT=8080
 ENV PGPORT=5432
 
+RUN mkdir /home/intermine/intermine
 COPY ./build.sh /home/intermine
-RUN chmod a+rx /home/intermine/build.sh
+RUN chown -R intermine:intermine /home/intermine
+RUN chmod u+x /home/intermine/build.sh
+
+RUN ls -l /home/intermine >> /tmp/log.log
+
 WORKDIR /home/intermine/intermine
 
+USER intermine
+
 CMD ["/bin/sh","/home/intermine/build.sh"]
diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index b6cd5e5..0debc65 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -1,24 +1,75 @@
+volumes:
+  dump_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/mine/dump"
+
+  configs_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/mine/configs"
+
+  packages_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/mine/packages"
+
+  dot_intermine_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/mine/intermine"
+
+  mine_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/mine/${MINE_NAME}"
+
+  postgres_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/postgres"
+
+  solr_volume:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: "./data/solr"
+
 services:
   intermine_builder:
     container_name: intermine_builder
     build:
       context: ./intermine_builder
       dockerfile: ./intermine_builder.Dockerfile
-    user: ${DOCKER_UID:-1000}:${DOCKER_GID:-1000}
+      args:
+        - USER_ID=${DOCKER_UID}
+        - GROUP_ID=${DOCKER_GID}
     volumes:
-      # - ./data/mine/data:/home/intermine/intermine/data
-      - ./data/mine/dump:/home/intermine/intermine/dump
-      - ./data/mine/configs:/home/intermine/intermine/configs
-      - ./data/mine/packages:/home/intermine/.m2
-      - ./data/mine/intermine:/home/intermine/.intermine
-      - ./data/mine/${MINE_NAME:-biotestmine}:/home/intermine/intermine/${MINE_NAME:-biotestmine}
+      - dump_volume:/home/intermine/intermine/dump
+      - configs_volume:/home/intermine/intermine/configs
+      - packages_volume:/home/intermine/.m2
+      - dot_intermine_volume:/home/intermine/.intermine
+      - mine_volume:/home/intermine/intermine/${MINE_NAME}
     environment:
-      - MINE_NAME=${MINE_NAME:-biotestmine}
-      - MINE_REPO_URL=${MINE_REPO_URL:-}
-      - IM_DATA_DIR=${IM_DATA_DIR:-DATA_DIR}
-      - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"}
-      - IM_REPO_URL=${IM_REPO_URL:-}
-      - IM_REPO_BRANCH=${IM_REPO_BRANCH:-}
+      - MINE_NAME=${MINE_NAME}
+      - MINE_REPO_URL=${MINE_REPO_URL}
+      - IM_DATA_DIR=${IM_DATA_DIR}
+      - MEM_OPTS=${MEM_OPTS}
+      - IM_REPO_URL=${IM_REPO_URL}
+      - IM_REPO_BRANCH=${IM_REPO_BRANCH}
     depends_on:
       - postgres
       - solr
@@ -29,9 +80,9 @@ services:
     build:
       context: ./postgres
       dockerfile: ./postgres.Dockerfile
-    user: ${DOCKER_UID:-1000}:${DOCKER_GID:-1000}
+    user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
-      - ./data/postgres:/var/lib/postgresql/data
+      - postgres_volume:/var/lib/postgresql/data
 
   solr:
     container_name: intermine_solr
@@ -39,11 +90,11 @@ services:
       context: ./solr
       dockerfile: ./solr.Dockerfile
     environment:
-      - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"}
-      - MINE_NAME=${MINE_NAME:-biotestmine}
-    user: ${DOCKER_UID:-1000}:${DOCKER_GID:-1000}
+      - MEM_OPTS=${MEM_OPTS}
+      - MINE_NAME=${MINE_NAME}
+    user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
-      - ./data/solr:/var/solr
+      - solr_volume:/var/solr
 
   tomcat:
     container_name: intermine_tomcat
@@ -51,12 +102,6 @@ services:
       context: ./tomcat
       dockerfile: ./tomcat.Dockerfile
     environment:
-      - MEM_OPTS=${MEM_OPTS:-"-Xmx1g -Xms500m"}
+      - MEM_OPTS=${MEM_OPTS}
     ports:
-      - ${TOMCAT_HOST_PORT:-9999}:${TOMCAT_PORT:-8080}
-
-# volumes:
-#   PostgresDBData:
-#   SolrDBData:
-#   MineData:
-#   MineConfigs:
+      - ${TOMCAT_HOST_PORT}:${TOMCAT_PORT}
diff --git a/mkdatadirs.sh b/mkdatadirs.sh
index 9a7be27..97fe97a 100755
--- a/mkdatadirs.sh
+++ b/mkdatadirs.sh
@@ -1,32 +1,9 @@
 #!/usr/bin/env sh
 
-## Parse a docker-compose config YAML file and create the directories listed
-## as volumes. This is useful since the directories need to exist for their
-## permissions to be preserved (otherwise the docker daemon would create them
-## as root). Doing this is the only way to have volume directories which are
-## owned by a user ID determined during container runtime, usually as the
-## logged in user so that you can share files between the user and the docker
-## container without running into permission problems.
-
-if [ -z $1 ]; then
-  echo "Usage: ${0} docker-compose.yml"
-  echo "This script creates the directories listed as volumes in the docker-compose config file passed as argument."
-  exit 1;
-fi
-
-# awk is the simplest way to parse a YAML without introducing any dependencies.
-awk 'BEGIN {FS="\n"} {
-  for (i=1; i<=NF; i++) {
-    if ($i ~ /^ *volumes:/) {
-      reading=1
-    } else if (reading) {
-      if ($i ~ /^ *-/) {
-        start=index($i, "-")+2
-        count=index($i, ":")-start
-        print substr($i, start, count)
-      } else if ($i !~ /^ *#/) {
-        reading=0
-      }
-    }
-  }
-}' $1 | xargs mkdir -p
+mkdir -p \
+    ./data/mine/dump \
+    ./data/mine/configs \
+    ./data/mine/packages \
+    ./data/mine/intermine \
+    ./data/postgres \
+    ./data/solr

From 3f5ca8e091fcbdc34c048380c351dcc3340bbc3f Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 25 Jul 2024 16:34:57 +0100
Subject: [PATCH 38/62] Fix mkdatadirs arguments

---
 .github/workflows/camchildmine.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/camchildmine.yml b/.github/workflows/camchildmine.yml
index 4060069..0f82f4f 100644
--- a/.github/workflows/camchildmine.yml
+++ b/.github/workflows/camchildmine.yml
@@ -26,7 +26,7 @@ jobs:
             - name: Make data dirs
               run: |
                   set -eux -o pipefail
-                  ./mkdatadirs.sh local.docker-compose.yml
+                  ./mkdatadirs.sh
             - name: Clone camCHILDMine
               run: |
                   set -eux -o pipefail

From b3451ae1cd7d71071b4ff1fdedab7652bd7b5c1d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 25 Jul 2024 16:46:42 +0100
Subject: [PATCH 39/62] Put default Docker settings in .env

---
 .env       | 10 ++++++++++
 .gitignore |  1 -
 2 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 .env

diff --git a/.env b/.env
new file mode 100644
index 0000000..6532052
--- /dev/null
+++ b/.env
@@ -0,0 +1,10 @@
+DOCKER_UID=1000
+DOCKER_GID=1000
+MINE_NAME=biotestmine
+MINE_REPO_URL=
+IM_DATA_DIR=DATA_DIR
+MEM_OPTS=:"-Xmx2g -Xms1g"
+IM_REPO_URL=
+IM_REPO_BRANCH=
+TOMCAT_HOST_PORT=9999
+TOMCAT_PORT=8080
diff --git a/.gitignore b/.gitignore
index bbbec6b..f85f93e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
-.env
 /data
 
 # Emacs backup

From f2dd6c7e1288f526f6104e0ce0362feb7aab8f19 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 25 Jul 2024 16:49:46 +0100
Subject: [PATCH 40/62] Trigger workflows when .env changes [no ci]

---
 .github/workflows/biotestmine.yml  | 1 +
 .github/workflows/camchildmine.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml
index f7eac98..0660758 100644
--- a/.github/workflows/biotestmine.yml
+++ b/.github/workflows/biotestmine.yml
@@ -5,6 +5,7 @@ name: Biotestmine
 on:
     push:
         paths:
+            - '.env'
             - '.github/workflows/biotestmine.yml'
             - 'intermine_builder/**'
             - 'local.docker-compose.yml'
diff --git a/.github/workflows/camchildmine.yml b/.github/workflows/camchildmine.yml
index 0f82f4f..c246dac 100644
--- a/.github/workflows/camchildmine.yml
+++ b/.github/workflows/camchildmine.yml
@@ -5,6 +5,7 @@ name: CamCHILDMine
 on:
     push:
         paths:
+            - '.env'
             - '.github/workflows/camchildmine.yml'
             - 'intermine_builder/**'
             - 'local.docker-compose.yml'

From 622be9e8103f87defea7c302f6b51eab8e55f61c Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 25 Jul 2024 17:08:09 +0100
Subject: [PATCH 41/62] Reminder to mkdir for your mine

---
 mkdatadirs.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mkdatadirs.sh b/mkdatadirs.sh
index 97fe97a..0083d4a 100755
--- a/mkdatadirs.sh
+++ b/mkdatadirs.sh
@@ -7,3 +7,5 @@ mkdir -p \
     ./data/mine/intermine \
     ./data/postgres \
     ./data/solr
+
+echo "Don't forget to also mkdir ./data/mine/<your mine name>"

From 32ea5dca9dcb2775acf9ecbc1708d4b44ebe6f53 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 31 Jul 2024 14:25:52 +0100
Subject: [PATCH 42/62] Wait for postgres to be available or fail

---
 intermine_builder/build.sh                     | 10 ++++------
 intermine_builder/intermine_builder.Dockerfile |  3 ++-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index d8b0038..53996a2 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -6,6 +6,7 @@ THE_MINE_NAME=${MINE_NAME:-biotestmine}
 FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0}
 IM_VERSION=${IM_VERSION:-}
 BIO_VERSION=${BIO_VERSION:-}
+THE_PGPORT=${PGPORT:-5432}
 
 if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then
     echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists"
@@ -100,7 +101,7 @@ if [ ! -f /home/intermine/.intermine/${THE_MINE_NAME}.properties ]; then
     sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${TOMCAT_HOST:-tomcat}:${TOMCAT_PORT:-8080}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
-    sed -i "s/serverName=localhost/serverName=${PGHOST:-postgres}:${PGPORT:-5432}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/serverName=localhost/serverName=${PGHOST:-postgres}:${THE_PGPORT:-5432}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
 
 
     # echo "project.rss=http://localhost:$WORDPRESS_PORT/?feed=rss2" >> /home/intermine/.intermine/${THE_MINE_NAME}.properties
@@ -156,11 +157,8 @@ fi
 echo "$(date +%Y/%m/%d-%H:%M) Connect and create Postgres databases" #>> /home/intermine/intermine/build.progress
 
 # # Wait for database
-# dockerize -wait tcp://postgres:$PGPORT -timeout 60s
-until psql -U postgres -h ${PGHOST:-postgres} -c '\l'; do
-  echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is unavailable - sleeping"
-  sleep 1
-done
+# dockerize -wait tcp://postgres:$THE_PGPORT -timeout 60s
+wait4x tcp postgres:5432 --timeout 120s
 echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is up - executing command"
 
 # Close all open connections to database
diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile
index 1288497..189eae3 100644
--- a/intermine_builder/intermine_builder.Dockerfile
+++ b/intermine_builder/intermine_builder.Dockerfile
@@ -22,7 +22,8 @@ RUN apk add --no-cache bash \
                        maven \
                        postgresql-client \
                        perl \
-                       perl-utils
+                       perl-utils \
+                       wait4x
 
 RUN apk add --no-cache build-base
 RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing perl-moosex

From c00a6258a0952fddf6243d37fd1cf80ddf2b7517 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 31 Jul 2024 14:59:21 +0100
Subject: [PATCH 43/62] Use wait-for-it instead of wait4x to wait for postgres

---
 intermine_builder/build.sh                    |   2 +-
 .../intermine_builder.Dockerfile              |   6 +-
 intermine_builder/wait-for-it.sh              | 182 ++++++++++++++++++
 3 files changed, 187 insertions(+), 3 deletions(-)
 create mode 100755 intermine_builder/wait-for-it.sh

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 53996a2..820168c 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -158,7 +158,7 @@ echo "$(date +%Y/%m/%d-%H:%M) Connect and create Postgres databases" #>> /home/i
 
 # # Wait for database
 # dockerize -wait tcp://postgres:$THE_PGPORT -timeout 60s
-wait4x tcp postgres:5432 --timeout 120s
+wait-for-it postgres:5432 -t 300
 echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is up - executing command"
 
 # Close all open connections to database
diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile
index 189eae3..a543b9c 100644
--- a/intermine_builder/intermine_builder.Dockerfile
+++ b/intermine_builder/intermine_builder.Dockerfile
@@ -22,8 +22,7 @@ RUN apk add --no-cache bash \
                        maven \
                        postgresql-client \
                        perl \
-                       perl-utils \
-                       wait4x
+                       perl-utils
 
 RUN apk add --no-cache build-base
 RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing perl-moosex
@@ -88,6 +87,9 @@ COPY ./build.sh /home/intermine
 RUN chown -R intermine:intermine /home/intermine
 RUN chmod u+x /home/intermine/build.sh
 
+COPY ./wait-for-it.sh /usr/local/bin/wait-for-it
+RUN chmod +x /usr/local/bin/wait-for-it
+
 RUN ls -l /home/intermine >> /tmp/log.log
 
 WORKDIR /home/intermine/intermine
diff --git a/intermine_builder/wait-for-it.sh b/intermine_builder/wait-for-it.sh
new file mode 100755
index 0000000..d990e0d
--- /dev/null
+++ b/intermine_builder/wait-for-it.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

From 6bdb8c1690bc7664a99f98d4ad7b955b68fa0e71 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 31 Jul 2024 15:37:12 +0100
Subject: [PATCH 44/62] Fix permissions problem with Postgres container

---
 intermine_builder/build.sh | 33 +++++++++++++++++----------------
 local.docker-compose.yml   |  4 ++++
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 820168c..7c477bf 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -7,6 +7,7 @@ FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0}
 IM_VERSION=${IM_VERSION:-}
 BIO_VERSION=${BIO_VERSION:-}
 THE_PGPORT=${PGPORT:-5432}
+THE_PGHOST=${PGHOST:-postgres}
 
 if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then
     echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists"
@@ -94,14 +95,14 @@ if [ ! -f /home/intermine/.intermine/${THE_MINE_NAME}.properties ]; then
 
     echo -e "$(date +%Y/%m/%d-%H:%M) Set properties in .intermine/${THE_MINE_NAME}.properties to\nPSQL_DB_NAME\tbiotestmine\nPSQL_USER\t$PSQL_USER\nPSQL_PWD\t$PSQL_PWD\nTOMCAT_USER\t$TOMCAT_USER\nTOMCAT_PWD\t$TOMCAT_PWD\nGRADLE_OPTS\t$GRADLE_OPTS" #>> /home/intermine/intermine/build.progress
 
-    #sed -i "s/PSQL_PORT/$PGPORT/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    #sed -i "s/PSQL_PORT/${THE_PGPORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/PSQL_DB_NAME/${THE_MINE_NAME}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/PSQL_USER/${PSQL_USER:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/PSQL_PWD/${PSQL_PWD:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${TOMCAT_HOST:-tomcat}:${TOMCAT_PORT:-8080}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
-    sed -i "s/serverName=localhost/serverName=${PGHOST:-postgres}:${THE_PGPORT:-5432}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/serverName=localhost/serverName=${THE_PGHOST}:${THE_PGPORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
 
 
     # echo "project.rss=http://localhost:$WORDPRESS_PORT/?feed=rss2" >> /home/intermine/.intermine/${THE_MINE_NAME}.properties
@@ -158,32 +159,32 @@ echo "$(date +%Y/%m/%d-%H:%M) Connect and create Postgres databases" #>> /home/i
 
 # # Wait for database
 # dockerize -wait tcp://postgres:$THE_PGPORT -timeout 60s
-wait-for-it postgres:5432 -t 300
+wait-for-it postgres:5432 -t 60
 echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is up - executing command"
 
 # Close all open connections to database
-psql -U postgres -h ${PGHOST:-postgres} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();"
+psql -U postgres -h ${THE_PGHOST} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();"
 
 echo "$(date +%Y/%m/%d-%H:%M) Database is now available ..." #>> /home/intermine/intermine/build.progress
 echo "$(date +%Y/%m/%d-%H:%M) Reset databases and roles" #>> /home/intermine/intermine/build.progress
 
 # Delete Databases if exist
-psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"${THE_MINE_NAME}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"items-${THE_MINE_NAME}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"userprofile-${THE_MINE_NAME}\";"
+psql -U postgres -h ${THE_PGHOST} -c "DROP DATABASE IF EXISTS \"${THE_MINE_NAME}\";"
+psql -U postgres -h ${THE_PGHOST} -c "DROP DATABASE IF EXISTS \"items-${THE_MINE_NAME}\";"
+psql -U postgres -h ${THE_PGHOST} -c "DROP DATABASE IF EXISTS \"userprofile-${THE_MINE_NAME}\";"
 
-# psql -U postgres -h ${PGHOST:-postgres} -c "DROP ROLE IF EXISTS ${PSQL_USER:-postgres};"
+# psql -U postgres -h ${THE_PGHOST} -c "DROP ROLE IF EXISTS ${PSQL_USER:-postgres};"
 
 # Create Databases
 echo "$(date +%Y/%m/%d-%H:%M) Creating postgres database tables and roles.." #>> /home/intermine/intermine/build.progress
-# psql -U postgres -h ${PGHOST:-postgres} -c "CREATE USER ${PSQL_USER:-postgres} WITH PASSWORD '${PSQL_PWD:-postgres}';"
-psql -U postgres -h ${PGHOST:-postgres} -c "ALTER USER ${PSQL_USER:-postgres} WITH SUPERUSER;"
-psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"${THE_MINE_NAME}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"items-${THE_MINE_NAME}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"userprofile-${THE_MINE_NAME}\";"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
-psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
+# psql -U postgres -h ${THE_PGHOST} -c "CREATE USER ${PSQL_USER:-postgres} WITH PASSWORD '${PSQL_PWD:-postgres}';"
+psql -U postgres -h ${THE_PGHOST} -c "ALTER USER ${PSQL_USER:-postgres} WITH SUPERUSER;"
+psql -U postgres -h ${THE_PGHOST} -c "CREATE DATABASE \"${THE_MINE_NAME}\";"
+psql -U postgres -h ${THE_PGHOST} -c "CREATE DATABASE \"items-${THE_MINE_NAME}\";"
+psql -U postgres -h ${THE_PGHOST} -c "CREATE DATABASE \"userprofile-${THE_MINE_NAME}\";"
+psql -U postgres -h ${THE_PGHOST} -c "GRANT ALL PRIVILEGES ON DATABASE \"${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
+psql -U postgres -h ${THE_PGHOST} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
+psql -U postgres -h ${THE_PGHOST} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};"
 
 
 cd ${THE_MINE_NAME}
diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index 0debc65..a789ad8 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -83,6 +83,10 @@ services:
     user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
       - postgres_volume:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - PGDATA=/var/lib/postgresql/data
 
   solr:
     container_name: intermine_solr

From 682322e9a7492eb9d6412d268c997f2b9fbe0a5d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 31 Jul 2024 16:34:14 +0100
Subject: [PATCH 45/62] Bail out early if there was a problem with Solr, Tomcat
 or Postgres

---
 intermine_builder/build.sh | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 7c477bf..79a7a85 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -6,8 +6,20 @@ THE_MINE_NAME=${MINE_NAME:-biotestmine}
 FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0}
 IM_VERSION=${IM_VERSION:-}
 BIO_VERSION=${BIO_VERSION:-}
-THE_PGPORT=${PGPORT:-5432}
+
 THE_PGHOST=${PGHOST:-postgres}
+THE_PGPORT=${PGPORT:-5432}
+
+THE_SOLR_HOST=${SOLR_HOST:-solr}
+THE_SOLR_PORT=${SOLR_PORT:-8983}
+
+THE_TOMCAT_HOST=${TOMCAT_HOST:-tomcat}
+THE_TOMCAT_PORT=${TOMCAT_PORT:-8080}
+
+# Bail out early if none of these is up
+wait-for-it ${THE_PGHOST}:${THE_PGPORT} -t 60
+wait-for-it ${THE_SOLR_HOST}:${THE_SOLR_PORT} -t 60
+wait-for-it ${THE_TOMCAT_HOST}:${THE_TOMCAT_PORT} -t 60
 
 if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then
     echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists"
@@ -62,7 +74,7 @@ else
     ls -l
     git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${THE_MINE_NAME}
     echo "$(date +%Y/%m/%d-%H:%M) Update keyword_search.properties to use http://solr" #>> /home/intermine/intermine/build.progress
-    sed -i 's/localhost/'${SOLR_HOST:-solr}'/g' ./${THE_MINE_NAME}/dbmodel/resources/keyword_search.properties
+    sed -i 's/localhost/'${THE_SOLR_HOST}'/g' ./${THE_MINE_NAME}/dbmodel/resources/keyword_search.properties
 fi
 
 # If InterMine or Bio versions have been set (likely because of a custom
@@ -101,7 +113,7 @@ if [ ! -f /home/intermine/.intermine/${THE_MINE_NAME}.properties ]; then
     sed -i "s/PSQL_PWD/${PSQL_PWD:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
-    sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${TOMCAT_HOST:-tomcat}:${TOMCAT_PORT:-8080}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
+    sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${THE_TOMCAT_HOST}:${THE_TOMCAT_PORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
     sed -i "s/serverName=localhost/serverName=${THE_PGHOST}:${THE_PGPORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties
 
 
@@ -157,9 +169,6 @@ fi
 
 echo "$(date +%Y/%m/%d-%H:%M) Connect and create Postgres databases" #>> /home/intermine/intermine/build.progress
 
-# # Wait for database
-# dockerize -wait tcp://postgres:$THE_PGPORT -timeout 60s
-wait-for-it postgres:5432 -t 60
 echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is up - executing command"
 
 # Close all open connections to database

From 0640b406f4ae04b83454a5a62b7df3597eef0535 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 31 Jul 2024 16:35:09 +0100
Subject: [PATCH 46/62] Run Docker and Postgres containers as root

There are permissions problems if we try to use a different user
---
 local.docker-compose.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index a789ad8..6036e3b 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -80,7 +80,6 @@ services:
     build:
       context: ./postgres
       dockerfile: ./postgres.Dockerfile
-    user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
       - postgres_volume:/var/lib/postgresql/data
     environment:
@@ -96,7 +95,6 @@ services:
     environment:
       - MEM_OPTS=${MEM_OPTS}
       - MINE_NAME=${MINE_NAME}
-    user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
       - solr_volume:/var/solr
 

From f0505241fe200be625dd57eb2c51d7894e7d4e85 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 1 Aug 2024 07:09:37 +0100
Subject: [PATCH 47/62] Try PGPASSWORD for postgres and reinstate user for Solr

---
 local.docker-compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index 6036e3b..4afafc3 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -70,6 +70,7 @@ services:
       - MEM_OPTS=${MEM_OPTS}
       - IM_REPO_URL=${IM_REPO_URL}
       - IM_REPO_BRANCH=${IM_REPO_BRANCH}
+      - PGPASSWORD=postgres
     depends_on:
       - postgres
       - solr
@@ -95,6 +96,7 @@ services:
     environment:
       - MEM_OPTS=${MEM_OPTS}
       - MINE_NAME=${MINE_NAME}
+    user: ${UID:-1000}:${GID:-1000}
     volumes:
       - solr_volume:/var/solr
 

From 7ee825b0eecace843d2030b21c2287e9ecbaf9c2 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 1 Aug 2024 07:18:35 +0100
Subject: [PATCH 48/62] Fix Solr user

---
 local.docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index 4afafc3..b5f5e2b 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -96,7 +96,7 @@ services:
     environment:
       - MEM_OPTS=${MEM_OPTS}
       - MINE_NAME=${MINE_NAME}
-    user: ${UID:-1000}:${GID:-1000}
+    user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
       - solr_volume:/var/solr
 

From 70420f6b5c1d5950b97ed68639eed1fb5b94fad1 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Fri, 2 Aug 2024 09:15:25 +0100
Subject: [PATCH 49/62] Ensure Solr volume is owned by the Solr user

Don't try to do anything with the Solr UID/GID
---
 local.docker-compose.yml | 1 -
 mkdatadirs.sh            | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index b5f5e2b..c9a60bd 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -96,7 +96,6 @@ services:
     environment:
       - MEM_OPTS=${MEM_OPTS}
       - MINE_NAME=${MINE_NAME}
-    user: ${DOCKER_UID}:${DOCKER_GID}
     volumes:
       - solr_volume:/var/solr
 
diff --git a/mkdatadirs.sh b/mkdatadirs.sh
index 0083d4a..9993972 100755
--- a/mkdatadirs.sh
+++ b/mkdatadirs.sh
@@ -8,4 +8,6 @@ mkdir -p \
     ./data/postgres \
     ./data/solr
 
+sudo chown -R 8983:8983 ./data/solr
+
 echo "Don't forget to also mkdir ./data/mine/<your mine name>"

From d2efdfabc8cca833d85c7bf5db237e5e1c72e541 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Fri, 2 Aug 2024 10:38:02 +0100
Subject: [PATCH 50/62] Use docker compose up now the build seems to be working

cargoDeployRemote fails if run for a second time
---
 .github/workflows/biotestmine.yml  | 12 ++++++------
 .github/workflows/camchildmine.yml | 11 +++++------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml
index 0660758..ff74e4c 100644
--- a/.github/workflows/biotestmine.yml
+++ b/.github/workflows/biotestmine.yml
@@ -27,21 +27,20 @@ jobs:
             - name: Make data dirs
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
                   ./mkdatadirs.sh
                   mkdir ./data/mine/biotestmine
-            - name: Run
-              run: |
-                  set -eux -o pipefail
-                  cd ${GITHUB_WORKSPACE}
-                  # Try to trap any build errors early
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml run --rm intermine_builder
             - name: Up
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
+                  # Use run to trap any build errors early
+                  # DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml run --rm intermine_builder
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up -d
             - name: Wait
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes
@@ -51,6 +50,7 @@ jobs:
               if: failure()
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
                   sleep 120
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres
diff --git a/.github/workflows/camchildmine.yml b/.github/workflows/camchildmine.yml
index c246dac..1b4abfe 100644
--- a/.github/workflows/camchildmine.yml
+++ b/.github/workflows/camchildmine.yml
@@ -35,19 +35,17 @@ jobs:
                   git clone https://github.com/ucam-department-of-psychiatry/camCHILDMine
                   cd camCHILDMine
                   git checkout cadre-dev
-            - name: Run
-              run: |
-                  set -eux -o pipefail
-                  cd ${GITHUB_WORKSPACE}
-                  # Try to trap any build errors early
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run --rm intermine_builder
             - name: Up
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
+                  # Use run to trap any build errors early
+                  # DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run --rm intermine_builder
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up -d
             - name: Wait
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
                   # Wait 15 minutes
@@ -57,6 +55,7 @@ jobs:
               if: failure()
               run: |
                   set -eux -o pipefail
+                  cd ${GITHUB_WORKSPACE}
                   sleep 120
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres

From 4156e9b10baa019ccd28b2a4382f938d1f765585 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Fri, 2 Aug 2024 10:49:12 +0100
Subject: [PATCH 51/62] Move checking the webapp so we dump the docker logs on
 failure

---
 .github/workflows/biotestmine.yml  | 9 +++++----
 .github/workflows/camchildmine.yml | 9 +++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml
index ff74e4c..453a438 100644
--- a/.github/workflows/biotestmine.yml
+++ b/.github/workflows/biotestmine.yml
@@ -46,6 +46,11 @@ jobs:
                   # Wait 15 minutes
                   wait-for-it localhost:9999 --timeout=900
 
+            - name: Check webapp
+              run: |
+                  set -eux -o pipefail
+                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"
+
             - name: Dump Docker logs
               if: failure()
               run: |
@@ -56,7 +61,3 @@ jobs:
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs solr
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs tomcat
-            - name: Check webapp
-              run: |
-                  set -eux -o pipefail
-                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"
diff --git a/.github/workflows/camchildmine.yml b/.github/workflows/camchildmine.yml
index 1b4abfe..024cd67 100644
--- a/.github/workflows/camchildmine.yml
+++ b/.github/workflows/camchildmine.yml
@@ -51,6 +51,11 @@ jobs:
                   # Wait 15 minutes
                   wait-for-it localhost:9999 --timeout=900
 
+            - name: Check webapp
+              run: |
+                  set -eux -o pipefail
+                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/camCHILDMine/"
+
             - name: Dump Docker logs
               if: failure()
               run: |
@@ -61,7 +66,3 @@ jobs:
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs solr
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs tomcat
-            - name: Check webapp
-              run: |
-                  set -eux -o pipefail
-                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/camCHILDMine/"

From b23960ba59f9322e2a4a06fe5a273f613e74943d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Fri, 2 Aug 2024 11:02:31 +0100
Subject: [PATCH 52/62] Fix camchildmine web app path

---
 .github/workflows/camchildmine.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/camchildmine.yml b/.github/workflows/camchildmine.yml
index 024cd67..4f3163f 100644
--- a/.github/workflows/camchildmine.yml
+++ b/.github/workflows/camchildmine.yml
@@ -54,7 +54,7 @@ jobs:
             - name: Check webapp
               run: |
                   set -eux -o pipefail
-                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/camCHILDMine/"
+                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/camchildmine/"
 
             - name: Dump Docker logs
               if: failure()

From 75fd43405f53483b2573e839fb6cf5cf0078162b Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Thu, 7 Nov 2024 14:27:04 +0000
Subject: [PATCH 53/62] Fix Postgres health check

---
 postgres/docker-healthcheck | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/docker-healthcheck b/postgres/docker-healthcheck
index 251c559..e086897 100755
--- a/postgres/docker-healthcheck
+++ b/postgres/docker-healthcheck
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eo pipefail
 
-host="${PGHOST:-intermine-postgres}"
+host="${PGHOST:-localhost}"
 port="${PGPORT:-5432}"
 user="${PSQL_USER:-postgres}"
 db="mine"

From 25106a867ec26d2d6520d67aa9acefe04e89db56 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Fri, 8 Nov 2024 14:26:52 +0000
Subject: [PATCH 54/62] Don't copy malaria-data.tar.gz if it doesn't exist

---
 intermine_builder/build.sh | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 79a7a85..63494ef 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -155,14 +155,16 @@ if [ -d /home/intermine/intermine/data ]; then
     fi
 else
     echo "$(date +%Y/%m/%d-%H:%M) No user data directory found"
+    mkdir -p /home/intermine/intermine/data/
     if [ ! -d /home/intermine/intermine/data/malaria ]; then
-        echo "$(date +%Y/%m/%d-%H:%M) Copy malaria-data to ~/data" #>> /home/intermine/intermine/build.progress
-        mkdir -p /home/intermine/intermine/data/
-        cp /home/intermine/intermine/${THE_MINE_NAME}/data/malaria-data.tar.gz /home/intermine/intermine/data/
-        cd /home/intermine/intermine/data/
-        tar -xf malaria-data.tar.gz
-        rm malaria-data.tar.gz
-        cd /home/intermine/intermine
+        if [ -f /home/intermine/intermine/${THE_MINE_NAME}/data/malaria-data.tar.gz ]; then
+            echo "$(date +%Y/%m/%d-%H:%M) Copy malaria-data to ~/data" #>> /home/intermine/intermine/build.progress
+            cp /home/intermine/intermine/${THE_MINE_NAME}/data/malaria-data.tar.gz /home/intermine/intermine/data/
+            cd /home/intermine/intermine/data/
+            tar -xf malaria-data.tar.gz
+            rm malaria-data.tar.gz
+            cd /home/intermine/intermine
+        fi
     fi
 fi
 

From 9423d7926670c85ae9f1e4c5e35319bd3437e0cd Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 10:09:13 +0000
Subject: [PATCH 55/62] Move CamCHILDMine workflow to that project

---
 .github/workflows/camchildmine.yml | 68 ------------------------------
 1 file changed, 68 deletions(-)
 delete mode 100644 .github/workflows/camchildmine.yml

diff --git a/.github/workflows/camchildmine.yml b/.github/workflows/camchildmine.yml
deleted file mode 100644
index 4f3163f..0000000
--- a/.github/workflows/camchildmine.yml
+++ /dev/null
@@ -1,68 +0,0 @@
----
-# yamllint disable rule:line-length
-name: CamCHILDMine
-# yamllint disable-line rule:truthy
-on:
-    push:
-        paths:
-            - '.env'
-            - '.github/workflows/camchildmine.yml'
-            - 'intermine_builder/**'
-            - 'local.docker-compose.yml'
-            - 'mkdatadirs.sh'
-            - 'postgres/**'
-            - 'solr/**'
-            - 'tomcat/**'
-
-jobs:
-    camchildmine:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v4
-
-            - name: Install prerequisites
-              run: |
-                  set -eux -o pipefail
-                  sudo apt -y install wait-for-it
-            - name: Make data dirs
-              run: |
-                  set -eux -o pipefail
-                  ./mkdatadirs.sh
-            - name: Clone camCHILDMine
-              run: |
-                  set -eux -o pipefail
-                  cd ${GITHUB_WORKSPACE}/data/mine
-                  git clone https://github.com/ucam-department-of-psychiatry/camCHILDMine
-                  cd camCHILDMine
-                  git checkout cadre-dev
-            - name: Up
-              run: |
-                  set -eux -o pipefail
-                  cd ${GITHUB_WORKSPACE}
-                  # Use run to trap any build errors early
-                  # DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml run --rm intermine_builder
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml up -d
-            - name: Wait
-              run: |
-                  set -eux -o pipefail
-                  cd ${GITHUB_WORKSPACE}
-                  # Wait for builder to finish and return exit code
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) MINE_NAME=camCHILDMine docker compose -f local.docker-compose.yml wait intermine_builder
-                  # Wait 15 minutes
-                  wait-for-it localhost:9999 --timeout=900
-
-            - name: Check webapp
-              run: |
-                  set -eux -o pipefail
-                  curl -I -L --retry 10 --fail --insecure "http://localhost:9999/camchildmine/"
-
-            - name: Dump Docker logs
-              if: failure()
-              run: |
-                  set -eux -o pipefail
-                  cd ${GITHUB_WORKSPACE}
-                  sleep 120
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs solr
-                  DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs tomcat

From df8b4fdfc41c34cde5f567252a440fa96a9821ec Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 10:14:26 +0000
Subject: [PATCH 56/62] Less noisy output of shell scripts

---
 .github/workflows/biotestmine.yml | 12 ++++++------
 intermine_builder/build.sh        |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml
index 453a438..690bc47 100644
--- a/.github/workflows/biotestmine.yml
+++ b/.github/workflows/biotestmine.yml
@@ -22,24 +22,24 @@ jobs:
 
             - name: Install prerequisites
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   sudo apt -y install wait-for-it
             - name: Make data dirs
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   cd ${GITHUB_WORKSPACE}
                   ./mkdatadirs.sh
                   mkdir ./data/mine/biotestmine
             - name: Up
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   cd ${GITHUB_WORKSPACE}
                   # Use run to trap any build errors early
                   # DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml run --rm intermine_builder
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up -d
             - name: Wait
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   cd ${GITHUB_WORKSPACE}
                   # Wait for builder to finish and return exit code
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml wait intermine_builder
@@ -48,13 +48,13 @@ jobs:
 
             - name: Check webapp
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/"
 
             - name: Dump Docker logs
               if: failure()
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   cd ${GITHUB_WORKSPACE}
                   sleep 120
                   DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder
diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 63494ef..4bde476 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -eux -o pipefail
+set -euo pipefail
 
 THE_MINE_NAME=${MINE_NAME:-biotestmine}
 FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0}

From 39841d879be11bccfdb8d94f21c0f32a35db617d Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 11:24:21 +0000
Subject: [PATCH 57/62] Fix mkdatadirs.sh to run from anywhere

---
 mkdatadirs.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/mkdatadirs.sh b/mkdatadirs.sh
index 9993972..2d2d370 100755
--- a/mkdatadirs.sh
+++ b/mkdatadirs.sh
@@ -1,13 +1,15 @@
 #!/usr/bin/env sh
 
+THIS_DIR=$(dirname $0)
+
 mkdir -p \
-    ./data/mine/dump \
-    ./data/mine/configs \
-    ./data/mine/packages \
-    ./data/mine/intermine \
-    ./data/postgres \
-    ./data/solr
+    ${THIS_DIR}/data/mine/dump \
+    ${THIS_DIR}/data/mine/configs \
+    ${THIS_DIR}/data/mine/packages \
+    ${THIS_DIR}/data/mine/intermine \
+    ${THIS_DIR}/data/postgres \
+    ${THIS_DIR}/data/solr
 
-sudo chown -R 8983:8983 ./data/solr
+sudo chown -R 8983:8983 ${THIS_DIR}/data/solr
 
-echo "Don't forget to also mkdir ./data/mine/<your mine name>"
+echo "Don't forget to also mkdir ${THIS_DIR}/data/mine/<your mine name>"

From 9ca3a087a5504116d28f1e6fcc417b9762118aac Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 11:42:42 +0000
Subject: [PATCH 58/62] Add note about potential 404 when building the webapp

---
 intermine_builder/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 4bde476..7f54a3d 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -26,6 +26,10 @@ if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE
     echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp"
     cd /home/intermine/intermine
     cd ${THE_MINE_NAME}
+    # If on opening the webapp you get the Tomcat error:
+    # HTTP Status 404 - /<yourmine>/ The requested resource is not available,
+    # implement the workaround at
+    # https://github.com/intermine/intermine/issues/2162#issuecomment-952099300
     ./gradlew cargoDeployRemote --stacktrace
     exit 0
 fi

From 1b26a48f51ef5c0ef7da1fad343c44125cbba7f9 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 11:45:55 +0000
Subject: [PATCH 59/62] Remove debug code in Dockerfile

---
 intermine_builder/intermine_builder.Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile
index a543b9c..4fc0f3e 100644
--- a/intermine_builder/intermine_builder.Dockerfile
+++ b/intermine_builder/intermine_builder.Dockerfile
@@ -90,8 +90,6 @@ RUN chmod u+x /home/intermine/build.sh
 COPY ./wait-for-it.sh /usr/local/bin/wait-for-it
 RUN chmod +x /usr/local/bin/wait-for-it
 
-RUN ls -l /home/intermine >> /tmp/log.log
-
 WORKDIR /home/intermine/intermine
 
 USER intermine

From 53e9856c51f1cc02724402c8dfe9f2209f78b856 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 11:52:50 +0000
Subject: [PATCH 60/62] Add licence for wait-for-it

---
 .../intermine_builder.Dockerfile              |  2 +-
 intermine_builder/wait-for-it/LICENSE         | 20 +++++++++++++++++++
 .../{ => wait-for-it}/wait-for-it.sh          |  0
 3 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 intermine_builder/wait-for-it/LICENSE
 rename intermine_builder/{ => wait-for-it}/wait-for-it.sh (100%)

diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile
index 4fc0f3e..b150218 100644
--- a/intermine_builder/intermine_builder.Dockerfile
+++ b/intermine_builder/intermine_builder.Dockerfile
@@ -87,7 +87,7 @@ COPY ./build.sh /home/intermine
 RUN chown -R intermine:intermine /home/intermine
 RUN chmod u+x /home/intermine/build.sh
 
-COPY ./wait-for-it.sh /usr/local/bin/wait-for-it
+COPY ./wait-for-it/wait-for-it.sh /usr/local/bin/wait-for-it
 RUN chmod +x /usr/local/bin/wait-for-it
 
 WORKDIR /home/intermine/intermine
diff --git a/intermine_builder/wait-for-it/LICENSE b/intermine_builder/wait-for-it/LICENSE
new file mode 100644
index 0000000..bd18d0c
--- /dev/null
+++ b/intermine_builder/wait-for-it/LICENSE
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/intermine_builder/wait-for-it.sh b/intermine_builder/wait-for-it/wait-for-it.sh
similarity index 100%
rename from intermine_builder/wait-for-it.sh
rename to intermine_builder/wait-for-it/wait-for-it.sh

From c9d9ab57fda9a36ee179af66483c9fe8dd915b78 Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 11:59:10 +0000
Subject: [PATCH 61/62] Use separate memory configuration for builder, Solr and
 Tomcat

---
 .env                     | 4 +++-
 local.docker-compose.yml | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.env b/.env
index 6532052..0dde61f 100644
--- a/.env
+++ b/.env
@@ -3,7 +3,9 @@ DOCKER_GID=1000
 MINE_NAME=biotestmine
 MINE_REPO_URL=
 IM_DATA_DIR=DATA_DIR
-MEM_OPTS=:"-Xmx2g -Xms1g"
+BUILDER_MEM_OPTS=:"-Xmx2g -Xms1g"
+SOLR_MEM_OPTS=:"-Xmx2g -Xms1g"
+TOMCAT_MEM_OPTS=:"-Xmx1g -Xms500m"
 IM_REPO_URL=
 IM_REPO_BRANCH=
 TOMCAT_HOST_PORT=9999
diff --git a/local.docker-compose.yml b/local.docker-compose.yml
index c9a60bd..9a03372 100644
--- a/local.docker-compose.yml
+++ b/local.docker-compose.yml
@@ -67,7 +67,7 @@ services:
       - MINE_NAME=${MINE_NAME}
       - MINE_REPO_URL=${MINE_REPO_URL}
       - IM_DATA_DIR=${IM_DATA_DIR}
-      - MEM_OPTS=${MEM_OPTS}
+      - MEM_OPTS=${BUILDER_MEM_OPTS}
       - IM_REPO_URL=${IM_REPO_URL}
       - IM_REPO_BRANCH=${IM_REPO_BRANCH}
       - PGPASSWORD=postgres
@@ -94,7 +94,7 @@ services:
       context: ./solr
       dockerfile: ./solr.Dockerfile
     environment:
-      - MEM_OPTS=${MEM_OPTS}
+      - MEM_OPTS=${SOLR_MEM_OPTS}
       - MINE_NAME=${MINE_NAME}
     volumes:
       - solr_volume:/var/solr
@@ -105,6 +105,6 @@ services:
       context: ./tomcat
       dockerfile: ./tomcat.Dockerfile
     environment:
-      - MEM_OPTS=${MEM_OPTS}
+      - MEM_OPTS=${TOMCAT_MEM_OPTS}
     ports:
       - ${TOMCAT_HOST_PORT}:${TOMCAT_PORT}

From fcf7844f5a09ceee90602d1596e25e513df6987e Mon Sep 17 00:00:00 2001
From: Martin Burchell <mb2353@cam.ac.uk>
Date: Wed, 18 Dec 2024 13:55:14 +0000
Subject: [PATCH 62/62] Remove debugging from build.sh

---
 intermine_builder/build.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh
index 7f54a3d..b79c727 100644
--- a/intermine_builder/build.sh
+++ b/intermine_builder/build.sh
@@ -73,9 +73,6 @@ if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ]; then
 else
     # echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}" #>> /home/intermine/intermine/build.progress
     echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}"
-    echo "User: $(id -u)"
-    echo "Group: $(id -g)"
-    ls -l
     git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${THE_MINE_NAME}
     echo "$(date +%Y/%m/%d-%H:%M) Update keyword_search.properties to use http://solr" #>> /home/intermine/intermine/build.progress
     sed -i 's/localhost/'${THE_SOLR_HOST}'/g' ./${THE_MINE_NAME}/dbmodel/resources/keyword_search.properties