diff --git a/.env b/.env new file mode 100644 index 0000000..0dde61f --- /dev/null +++ b/.env @@ -0,0 +1,12 @@ +DOCKER_UID=1000 +DOCKER_GID=1000 +MINE_NAME=biotestmine +MINE_REPO_URL= +IM_DATA_DIR=DATA_DIR +BUILDER_MEM_OPTS=:"-Xmx2g -Xms1g" +SOLR_MEM_OPTS=:"-Xmx2g -Xms1g" +TOMCAT_MEM_OPTS=:"-Xmx1g -Xms500m" +IM_REPO_URL= +IM_REPO_BRANCH= +TOMCAT_HOST_PORT=9999 +TOMCAT_PORT=8080 diff --git a/.github/workflows/biotestmine.yml b/.github/workflows/biotestmine.yml new file mode 100644 index 0000000..690bc47 --- /dev/null +++ b/.github/workflows/biotestmine.yml @@ -0,0 +1,63 @@ +--- +# yamllint disable rule:line-length +name: Biotestmine +# yamllint disable-line rule:truthy +on: + push: + paths: + - '.env' + - '.github/workflows/biotestmine.yml' + - 'intermine_builder/**' + - 'local.docker-compose.yml' + - 'mkdatadirs.sh' + - 'postgres/**' + - 'solr/**' + - 'tomcat/**' + +jobs: + biotestmine: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install prerequisites + run: | + set -euo pipefail + sudo apt -y install wait-for-it + - name: Make data dirs + run: | + set -euo pipefail + cd ${GITHUB_WORKSPACE} + ./mkdatadirs.sh + mkdir ./data/mine/biotestmine + - name: Up + run: | + set -euo pipefail + cd ${GITHUB_WORKSPACE} + # Use run to trap any build errors early + # DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml run --rm intermine_builder + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml up -d + - name: Wait + run: | + set -euo pipefail + cd ${GITHUB_WORKSPACE} + # Wait for builder to finish and return exit code + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml wait intermine_builder + # Wait 15 minutes + wait-for-it localhost:9999 --timeout=900 + + - name: Check webapp + run: | + set -euo pipefail + curl -I -L --retry 10 --fail --insecure "http://localhost:9999/biotestmine/" + + - name: Dump Docker logs + if: failure() + run: | + set -euo pipefail + cd ${GITHUB_WORKSPACE} + sleep 120 + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs intermine_builder + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs postgres + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs solr + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) docker compose -f local.docker-compose.yml logs tomcat diff --git a/.gitignore b/.gitignore index fd80175..f85f93e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ -.env /data + +# Emacs backup +*~ + +# Silver searcher +.ignore diff --git a/dockerhub.docker-compose.yml b/dockerhub.docker-compose.yml index b6a2929..9070e40 100644 --- a/dockerhub.docker-compose.yml +++ b/dockerhub.docker-compose.yml @@ -1,4 +1,3 @@ -version: "3" services: intermine_builder: container_name: intermine_builder @@ -19,7 +18,7 @@ services: - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"} - IM_REPO_URL=${IM_REPO_URL:-} - IM_REPO_BRANCH=${IM_REPO_BRANCH:-} - depends_on: + depends_on: - postgres - solr - tomcat diff --git a/intermine_builder/build.sh b/intermine_builder/build.sh index 0879302..b79c727 100644 --- a/intermine_builder/build.sh +++ b/intermine_builder/build.sh @@ -1,18 +1,39 @@ #!/bin/bash -if [ -d ${MINE_NAME:-biotestmine} ] && [ ! -z "$(ls -A ${MINE_NAME:-biotestmine})" ] && [ ! $FORCE_MINE_BUILD ]; then - echo "$(date +%Y/%m/%d-%H:%M) Mine already exists" +set -euo pipefail + +THE_MINE_NAME=${MINE_NAME:-biotestmine} +FORCE_MINE_BUILD=${FORCE_MINE_BUILD:-0} +IM_VERSION=${IM_VERSION:-} +BIO_VERSION=${BIO_VERSION:-} + +THE_PGHOST=${PGHOST:-postgres} +THE_PGPORT=${PGPORT:-5432} + +THE_SOLR_HOST=${SOLR_HOST:-solr} +THE_SOLR_PORT=${SOLR_PORT:-8983} + +THE_TOMCAT_HOST=${TOMCAT_HOST:-tomcat} +THE_TOMCAT_PORT=${TOMCAT_PORT:-8080} + +# Bail out early if none of these is up +wait-for-it ${THE_PGHOST}:${THE_PGPORT} -t 60 +wait-for-it ${THE_SOLR_HOST}:${THE_SOLR_PORT} -t 60 +wait-for-it ${THE_TOMCAT_HOST}:${THE_TOMCAT_PORT} -t 60 + +if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ] && [ ! $FORCE_MINE_BUILD ]; then + echo "$(date +%Y/%m/%d-%H:%M) Mine ${THE_MINE_NAME} already exists" echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp" cd /home/intermine/intermine - cd ${MINE_NAME:-biotestmine} - ./gradlew cargoDeployRemote - sleep 60 - ./gradlew cargoRedeployRemote --stacktrace + cd ${THE_MINE_NAME} + # If on opening the webapp you get the Tomcat error: + # HTTP Status 404 - // The requested resource is not available, + # implement the workaround at + # https://github.com/intermine/intermine/issues/2162#issuecomment-952099300 + ./gradlew cargoDeployRemote --stacktrace exit 0 fi -set -e - cd /home/intermine/intermine # Empty log @@ -44,84 +65,84 @@ fi echo "Starting mine build" echo $MINE_REPO_URL # Check if mine exists and is not empty -if [ -d ${MINE_NAME:-biotestmine} ] && [ ! -z "$(ls -A ${MINE_NAME:-biotestmine})" ]; then - echo "$(date +%Y/%m/%d-%H:%M) Update ${MINE_NAME:-biotestmine} to newest version" #>> /home/intermine/intermine/build.progress - cd ${MINE_NAME:-biotestmine} +if [ -d ${THE_MINE_NAME} ] && [ ! -z "$(ls -A ${THE_MINE_NAME})" ]; then + echo "$(date +%Y/%m/%d-%H:%M) Update ${THE_MINE_NAME} to newest version" #>> /home/intermine/intermine/build.progress + cd ${THE_MINE_NAME} # git pull cd /home/intermine/intermine else - # echo "$(date +%Y/%m/%d-%H:%M) Clone ${MINE_NAME:-biotestmine}" #>> /home/intermine/intermine/build.progress - echo "$(date +%Y/%m/%d-%H:%M) Clone ${MINE_NAME:-biotestmine}" - git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${MINE_NAME:-biotestmine} + # echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}" #>> /home/intermine/intermine/build.progress + echo "$(date +%Y/%m/%d-%H:%M) Clone ${THE_MINE_NAME}" + git clone ${MINE_REPO_URL:-https://github.com/intermine/biotestmine} ${THE_MINE_NAME} echo "$(date +%Y/%m/%d-%H:%M) Update keyword_search.properties to use http://solr" #>> /home/intermine/intermine/build.progress - sed -i 's/localhost/'${SOLR_HOST:-solr}'/g' ./${MINE_NAME:-biotestmine}/dbmodel/resources/keyword_search.properties + sed -i 's/localhost/'${THE_SOLR_HOST}'/g' ./${THE_MINE_NAME}/dbmodel/resources/keyword_search.properties fi # If InterMine or Bio versions have been set (likely because of a custom # InterMine build), update gradle.properties in the mine. if [ ! -z ${IM_VERSION} ]; then - sed -i "s/\(systemProp\.imVersion=\).*\$/\1${IM_VERSION}/" /home/intermine/intermine/${MINE_NAME:-biotestmine}/gradle.properties + sed -i "s/\(systemProp\.imVersion=\).*\$/\1${IM_VERSION}/" /home/intermine/intermine/${THE_MINE_NAME}/gradle.properties fi if [ ! -z ${BIO_VERSION} ]; then - sed -i "s/\(systemProp\.bioVersion=\).*\$/\1${BIO_VERSION}/" /home/intermine/intermine/${MINE_NAME:-biotestmine}/gradle.properties + sed -i "s/\(systemProp\.bioVersion=\).*\$/\1${BIO_VERSION}/" /home/intermine/intermine/${THE_MINE_NAME}/gradle.properties fi # Copy project_build from intermine_scripts repo -if [ ! -f /home/intermine/intermine/${MINE_NAME:-biotestmine}/project_build ]; then +if [ ! -f /home/intermine/intermine/${THE_MINE_NAME}/project_build ]; then echo "$(date +%Y/%m/%d-%H:%M) Cloning intermine scripts repo to /home/intermine/intermine/intermine-scripts" git clone https://github.com/intermine/intermine-scripts - echo "$(date +%Y/%m/%d-%H:%M) Copy project_build to /home/intermine/intermine/${MINE_NAME:-biotestmine}" - cp /home/intermine/intermine/intermine-scripts/project_build /home/intermine/intermine/${MINE_NAME:-biotestmine}/project_build - chmod +x /home/intermine/intermine/${MINE_NAME:-biotestmine}/project_build + echo "$(date +%Y/%m/%d-%H:%M) Copy project_build to /home/intermine/intermine/${THE_MINE_NAME}" + cp /home/intermine/intermine/intermine-scripts/project_build /home/intermine/intermine/${THE_MINE_NAME}/project_build + chmod +x /home/intermine/intermine/${THE_MINE_NAME}/project_build fi # Copy mine properties -if [ ! -f /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties ]; then - if [ ! -f /home/intermine/intermine/configs/${MINE_NAME:-biotestmine}.properties ]; then - echo "$(date +%Y/%m/%d-%H:%M) Copy biotestmine.properties to ~/.intermine/${MINE_NAME:-biotestmine}.properties" #>> /home/intermine/intermine/build.progress - cp /home/intermine/intermine/${MINE_NAME:-biotestmine}/data/${MINE_NAME:-biotestmine}.properties /home/intermine/.intermine/ +if [ ! -f /home/intermine/.intermine/${THE_MINE_NAME}.properties ]; then + if [ ! -f /home/intermine/intermine/configs/${THE_MINE_NAME}.properties ]; then + echo "$(date +%Y/%m/%d-%H:%M) Copy ${THE_MINE_NAME}.properties to ~/.intermine/${THE_MINE_NAME}.properties" #>> /home/intermine/intermine/build.progress + cp /home/intermine/intermine/${THE_MINE_NAME}/data/${THE_MINE_NAME}.properties /home/intermine/.intermine/ else - echo "$(date +%Y/%m/%d-%H:%M) Copy ${MINE_NAME:-biotestmine}.properties to ~/.intermine/${MINE_NAME:-biotestmine}.properties" - cp /home/intermine/intermine/configs/${MINE_NAME:-biotestmine}.properties /home/intermine/.intermine/ + echo "$(date +%Y/%m/%d-%H:%M) Copy ${THE_MINE_NAME}.properties to ~/.intermine/${THE_MINE_NAME}.properties" + cp /home/intermine/intermine/configs/${THE_MINE_NAME}.properties /home/intermine/.intermine/ fi - echo -e "$(date +%Y/%m/%d-%H:%M) Set properties in .intermine/${MINE_NAME:-biotestmine}.properties to\nPSQL_DB_NAME\tbiotestmine\nPSQL_USER\t$PSQL_USER\nPSQL_PWD\t$PSQL_PWD\nTOMCAT_USER\t$TOMCAT_USER\nTOMCAT_PWD\t$TOMCAT_PWD\nGRADLE_OPTS\t$GRADLE_OPTS" #>> /home/intermine/intermine/build.progress + echo -e "$(date +%Y/%m/%d-%H:%M) Set properties in .intermine/${THE_MINE_NAME}.properties to\nPSQL_DB_NAME\tbiotestmine\nPSQL_USER\t$PSQL_USER\nPSQL_PWD\t$PSQL_PWD\nTOMCAT_USER\t$TOMCAT_USER\nTOMCAT_PWD\t$TOMCAT_PWD\nGRADLE_OPTS\t$GRADLE_OPTS" #>> /home/intermine/intermine/build.progress - #sed -i "s/PSQL_PORT/$PGPORT/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/PSQL_DB_NAME/${MINE_NAME:-biotestmine}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/PSQL_USER/${PSQL_USER:-postgres}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/PSQL_PWD/${PSQL_PWD:-postgres}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${TOMCAT_HOST:-tomcat}:${TOMCAT_PORT:-8080}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - sed -i "s/serverName=localhost/serverName=${PGHOST:-postgres}:${PGPORT:-5432}/g" /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties + #sed -i "s/PSQL_PORT/${THE_PGPORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/PSQL_DB_NAME/${THE_MINE_NAME}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/PSQL_USER/${PSQL_USER:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/PSQL_PWD/${PSQL_PWD:-postgres}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/TOMCAT_USER/${TOMCAT_USER:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/TOMCAT_PWD/${TOMCAT_PWD:-tomcat}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/webapp.deploy.url=http:\/\/localhost:8080/webapp.deploy.url=http:\/\/${THE_TOMCAT_HOST}:${THE_TOMCAT_PORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties + sed -i "s/serverName=localhost/serverName=${THE_PGHOST}:${THE_PGPORT}/g" /home/intermine/.intermine/${THE_MINE_NAME}.properties - # echo "project.rss=http://localhost:$WORDPRESS_PORT/?feed=rss2" >> /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties - # echo "links.blog=https://localhost:$WORDPRESS_PORT" >> /home/intermine/.intermine/${MINE_NAME:-biotestmine}.properties + # echo "project.rss=http://localhost:$WORDPRESS_PORT/?feed=rss2" >> /home/intermine/.intermine/${THE_MINE_NAME}.properties + # echo "links.blog=https://localhost:$WORDPRESS_PORT" >> /home/intermine/.intermine/${THE_MINE_NAME}.properties fi # Copy mine configs -if [ ! -f /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml ]; then +if [ ! -f /home/intermine/intermine/${THE_MINE_NAME}/project.xml ]; then if [ -f /home/intermine/intermine/configs/project.xml ]; then - echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/${MINE_NAME:-biotestmine}/project.xml" - cp /home/intermine/intermine/configs/project.xml /home/intermine/intermine/${MINE_NAME:-biotestmine}/ + echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/${THE_MINE_NAME}/project.xml" + cp /home/intermine/intermine/configs/project.xml /home/intermine/intermine/${THE_MINE_NAME}/ echo "$(date +%Y/%m/%d-%H:%M) Set correct source path in project.xml" - sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml - sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml + sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml + sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml else - echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/biotestmine/project.xml" #>> /home/intermine/intermine/build.progress - cp /home/intermine/intermine/biotestmine/data/project.xml /home/intermine/intermine/biotestmine/ + echo "$(date +%Y/%m/%d-%H:%M) Copy project.xml to ~/intermine/${THE_MINE_NAME}/project.xml" #>> /home/intermine/intermine/build.progress + cp /home/intermine/intermine/${THE_MINE_NAME}/data/project.xml /home/intermine/intermine/${THE_MINE_NAME} echo "$(date +%Y/%m/%d-%H:%M) Set correct source path in project.xml" #>> /home/intermine/intermine/build.progress - sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/biotestmine/project.xml - sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/biotestmine/project.xml + sed -i 's/'${IM_DATA_DIR:-DATA_DIR}'/\/home\/intermine\/intermine\/data/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml + sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml fi else echo "$(date +%Y/%m/%d-%H:%M) Set correct source path in project.xml" - sed -i "s~${IM_DATA_DIR:-DATA_DIR}~/home/intermine/intermine/data~g" /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml - sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${MINE_NAME:-biotestmine}/project.xml + sed -i "s~${IM_DATA_DIR:-DATA_DIR}~/home/intermine/intermine/data~g" /home/intermine/intermine/${THE_MINE_NAME}/project.xml + sed -i 's/dump="true"/dump="false"/g' /home/intermine/intermine/${THE_MINE_NAME}/project.xml fi # Copy data @@ -135,54 +156,50 @@ if [ -d /home/intermine/intermine/data ]; then fi else echo "$(date +%Y/%m/%d-%H:%M) No user data directory found" + mkdir -p /home/intermine/intermine/data/ if [ ! -d /home/intermine/intermine/data/malaria ]; then - echo "$(date +%Y/%m/%d-%H:%M) Copy malria-data to ~/data" #>> /home/intermine/intermine/build.progress - mkdir -p /home/intermine/intermine/data/ - cp /home/intermine/intermine/biotestmine/data/malaria-data.tar.gz /home/intermine/intermine/data/ - cd /home/intermine/intermine/data/ - tar -xf malaria-data.tar.gz - rm malaria-data.tar.gz - cd /home/intermine/intermine + if [ -f /home/intermine/intermine/${THE_MINE_NAME}/data/malaria-data.tar.gz ]; then + echo "$(date +%Y/%m/%d-%H:%M) Copy malaria-data to ~/data" #>> /home/intermine/intermine/build.progress + cp /home/intermine/intermine/${THE_MINE_NAME}/data/malaria-data.tar.gz /home/intermine/intermine/data/ + cd /home/intermine/intermine/data/ + tar -xf malaria-data.tar.gz + rm malaria-data.tar.gz + cd /home/intermine/intermine + fi fi fi echo "$(date +%Y/%m/%d-%H:%M) Connect and create Postgres databases" #>> /home/intermine/intermine/build.progress -# # Wait for database -# dockerize -wait tcp://postgres:$PGPORT -timeout 60s -until psql -U postgres -h ${PGHOST:-postgres} -c '\l'; do - echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is unavailable - sleeping" - sleep 1 -done echo >&2 "$(date +%Y%m%dt%H%M%S) Postgres is up - executing command" # Close all open connections to database -psql -U postgres -h ${PGHOST:-postgres} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" +psql -U postgres -h ${THE_PGHOST} -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" echo "$(date +%Y/%m/%d-%H:%M) Database is now available ..." #>> /home/intermine/intermine/build.progress echo "$(date +%Y/%m/%d-%H:%M) Reset databases and roles" #>> /home/intermine/intermine/build.progress # Delete Databases if exist -psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"${MINE_NAME:-biotestmine}\";" -psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"items-${MINE_NAME:-biotestmine}\";" -psql -U postgres -h ${PGHOST:-postgres} -c "DROP DATABASE IF EXISTS \"userprofile-${MINE_NAME:-biotestmine}\";" +psql -U postgres -h ${THE_PGHOST} -c "DROP DATABASE IF EXISTS \"${THE_MINE_NAME}\";" +psql -U postgres -h ${THE_PGHOST} -c "DROP DATABASE IF EXISTS \"items-${THE_MINE_NAME}\";" +psql -U postgres -h ${THE_PGHOST} -c "DROP DATABASE IF EXISTS \"userprofile-${THE_MINE_NAME}\";" -# psql -U postgres -h ${PGHOST:-postgres} -c "DROP ROLE IF EXISTS ${PSQL_USER:-postgres};" +# psql -U postgres -h ${THE_PGHOST} -c "DROP ROLE IF EXISTS ${PSQL_USER:-postgres};" # Create Databases echo "$(date +%Y/%m/%d-%H:%M) Creating postgres database tables and roles.." #>> /home/intermine/intermine/build.progress -# psql -U postgres -h ${PGHOST:-postgres} -c "CREATE USER ${PSQL_USER:-postgres} WITH PASSWORD '${PSQL_PWD:-postgres}';" -psql -U postgres -h ${PGHOST:-postgres} -c "ALTER USER ${PSQL_USER:-postgres} WITH SUPERUSER;" -psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"${MINE_NAME:-biotestmine}\";" -psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"items-${MINE_NAME:-biotestmine}\";" -psql -U postgres -h ${PGHOST:-postgres} -c "CREATE DATABASE \"userprofile-${MINE_NAME:-biotestmine}\";" -psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE ${MINE_NAME:-biotestmine} to ${PSQL_USER:-postgres};" -psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${MINE_NAME:-biotestmine}\" to ${PSQL_USER:-postgres};" -psql -U postgres -h ${PGHOST:-postgres} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${MINE_NAME:-biotestmine}\" to ${PSQL_USER:-postgres};" +# psql -U postgres -h ${THE_PGHOST} -c "CREATE USER ${PSQL_USER:-postgres} WITH PASSWORD '${PSQL_PWD:-postgres}';" +psql -U postgres -h ${THE_PGHOST} -c "ALTER USER ${PSQL_USER:-postgres} WITH SUPERUSER;" +psql -U postgres -h ${THE_PGHOST} -c "CREATE DATABASE \"${THE_MINE_NAME}\";" +psql -U postgres -h ${THE_PGHOST} -c "CREATE DATABASE \"items-${THE_MINE_NAME}\";" +psql -U postgres -h ${THE_PGHOST} -c "CREATE DATABASE \"userprofile-${THE_MINE_NAME}\";" +psql -U postgres -h ${THE_PGHOST} -c "GRANT ALL PRIVILEGES ON DATABASE \"${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};" +psql -U postgres -h ${THE_PGHOST} -c "GRANT ALL PRIVILEGES ON DATABASE \"items-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};" +psql -U postgres -h ${THE_PGHOST} -c "GRANT ALL PRIVILEGES ON DATABASE \"userprofile-${THE_MINE_NAME}\" to ${PSQL_USER:-postgres};" -cd ${MINE_NAME:-biotestmine} +cd ${THE_MINE_NAME} echo "$(date +%Y/%m/%d-%H:%M) Running project_build script" ./project_build -b -T localhost /home/intermine/intermine/dump/dump @@ -213,6 +230,8 @@ echo "$(date +%Y/%m/%d-%H:%M) Gradle: build userDB" #>> /home/intermine/intermin echo "$(date +%Y/%m/%d-%H:%M) Gradle: build webapp" #>> /home/intermine/intermine/build.progress # ./gradlew clean -./gradlew cargoDeployRemote -sleep 60 -./gradlew cargoRedeployRemote --stacktrace #>> /home/intermine/intermine/build.progress +# --stacktrace --debug --info --scan +./gradlew cargoDeployRemote --stacktrace + +# Debug: Keep the container going +# tail -f /dev/null diff --git a/intermine_builder/intermine_builder.Dockerfile b/intermine_builder/intermine_builder.Dockerfile index 4dab998..b150218 100644 --- a/intermine_builder/intermine_builder.Dockerfile +++ b/intermine_builder/intermine_builder.Dockerfile @@ -1,12 +1,24 @@ FROM alpine:3.12.5 LABEL maintainer="Ank" +# ----------------------------------------------------------------------------- +# Permissions +# ----------------------------------------------------------------------------- +# https://vsupalov.com/docker-shared-permissions/ + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid $GROUP_ID intermine +RUN adduser -D -g '' -u $USER_ID -G intermine intermine + ENV JAVA_HOME="/usr/lib/jvm/default-jvm" RUN apk add --no-cache openjdk8 openjdk8-jre && \ ln -sf "${JAVA_HOME}/bin/"* "/usr/bin/" -RUN apk add --no-cache git \ +RUN apk add --no-cache bash \ + git \ maven \ postgresql-client \ perl \ @@ -58,9 +70,6 @@ RUN cpanm --force Ouch \ # Digest::MD5 \ # Log::Handler -RUN mkdir /home/intermine && mkdir /home/intermine/intermine -RUN chmod -R 777 /home/intermine - ENV MEM_OPTS="-Xmx1g -Xms500m" ENV GRADLE_OPTS="-server ${MEM_OPTS} -XX:+UseParallelGC -XX:SoftRefLRUPolicyMSPerMB=1 -XX:MaxHeapFreeRatio=99 -Dorg.gradle.daemon=false -Duser.home=/home/intermine" ENV HOME="/home/intermine" @@ -73,8 +82,16 @@ ENV TOMCAT_PWD="tomcat" ENV TOMCAT_PORT=8080 ENV PGPORT=5432 +RUN mkdir /home/intermine/intermine COPY ./build.sh /home/intermine -RUN chmod a+rx /home/intermine/build.sh +RUN chown -R intermine:intermine /home/intermine +RUN chmod u+x /home/intermine/build.sh + +COPY ./wait-for-it/wait-for-it.sh /usr/local/bin/wait-for-it +RUN chmod +x /usr/local/bin/wait-for-it + WORKDIR /home/intermine/intermine +USER intermine + CMD ["/bin/sh","/home/intermine/build.sh"] diff --git a/intermine_builder/wait-for-it/LICENSE b/intermine_builder/wait-for-it/LICENSE new file mode 100644 index 0000000..bd18d0c --- /dev/null +++ b/intermine_builder/wait-for-it/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/intermine_builder/wait-for-it/wait-for-it.sh b/intermine_builder/wait-for-it/wait-for-it.sh new file mode 100755 index 0000000..d990e0d --- /dev/null +++ b/intermine_builder/wait-for-it/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/local.docker-compose.yml b/local.docker-compose.yml index 8e057d5..9a03372 100644 --- a/local.docker-compose.yml +++ b/local.docker-compose.yml @@ -1,27 +1,77 @@ -version: "3" +volumes: + dump_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/mine/dump" + + configs_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/mine/configs" + + packages_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/mine/packages" + + dot_intermine_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/mine/intermine" + + mine_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/mine/${MINE_NAME}" + + postgres_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/postgres" + + solr_volume: + driver: local + driver_opts: + o: bind + type: none + device: "./data/solr" + services: intermine_builder: container_name: intermine_builder build: context: ./intermine_builder dockerfile: ./intermine_builder.Dockerfile - user: ${UID:-1000}:${GID:-1000} + args: + - USER_ID=${DOCKER_UID} + - GROUP_ID=${DOCKER_GID} volumes: - # - ./data/mine/data:/home/intermine/intermine/data - - ./data/mine/dump:/home/intermine/intermine/dump - - ./data/mine/configs:/home/intermine/intermine/configs - - ./data/mine/packages:/home/intermine/.m2 - - ./data/mine/intermine:/home/intermine/.intermine - # - ./data/mine/[PUT_YOUR_MINE_NAME_HERE]:/home/intermine/intermine/[PUT_YOUR_MINE_NAME_HERE] - - ./data/mine/biotestmine:/home/intermine/intermine/biotestmine + - dump_volume:/home/intermine/intermine/dump + - configs_volume:/home/intermine/intermine/configs + - packages_volume:/home/intermine/.m2 + - dot_intermine_volume:/home/intermine/.intermine + - mine_volume:/home/intermine/intermine/${MINE_NAME} environment: - - MINE_NAME=${MINE_NAME:-biotestmine} - - MINE_REPO_URL=${MINE_REPO_URL:-} - - IM_DATA_DIR=${IM_DATA_DIR:-DATA_DIR} - - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"} - - IM_REPO_URL=${IM_REPO_URL:-} - - IM_REPO_BRANCH=${IM_REPO_BRANCH:-} - depends_on: + - MINE_NAME=${MINE_NAME} + - MINE_REPO_URL=${MINE_REPO_URL} + - IM_DATA_DIR=${IM_DATA_DIR} + - MEM_OPTS=${BUILDER_MEM_OPTS} + - IM_REPO_URL=${IM_REPO_URL} + - IM_REPO_BRANCH=${IM_REPO_BRANCH} + - PGPASSWORD=postgres + depends_on: - postgres - solr - tomcat @@ -31,9 +81,12 @@ services: build: context: ./postgres dockerfile: ./postgres.Dockerfile - user: ${UID:-1000}:${GID:-1000} volumes: - - ./data/postgres:/var/lib/postgresql/data + - postgres_volume:/var/lib/postgresql/data + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - PGDATA=/var/lib/postgresql/data solr: container_name: intermine_solr @@ -41,11 +94,10 @@ services: context: ./solr dockerfile: ./solr.Dockerfile environment: - - MEM_OPTS=${MEM_OPTS:-"-Xmx2g -Xms1g"} - - MINE_NAME=${MINE_NAME:-biotestmine} - user: ${UID:-1000}:${GID:-1000} + - MEM_OPTS=${SOLR_MEM_OPTS} + - MINE_NAME=${MINE_NAME} volumes: - - ./data/solr:/var/solr + - solr_volume:/var/solr tomcat: container_name: intermine_tomcat @@ -53,12 +105,6 @@ services: context: ./tomcat dockerfile: ./tomcat.Dockerfile environment: - - MEM_OPTS=${MEM_OPTS:-"-Xmx1g -Xms500m"} + - MEM_OPTS=${TOMCAT_MEM_OPTS} ports: - - ${TOMCAT_HOST_PORT:-9999}:${TOMCAT_PORT:-8080} - -# volumes: -# PostgresDBData: -# SolrDBData: -# MineData: -# MineConfigs: + - ${TOMCAT_HOST_PORT}:${TOMCAT_PORT} diff --git a/mkdatadirs.sh b/mkdatadirs.sh index 9a7be27..2d2d370 100755 --- a/mkdatadirs.sh +++ b/mkdatadirs.sh @@ -1,32 +1,15 @@ #!/usr/bin/env sh -## Parse a docker-compose config YAML file and create the directories listed -## as volumes. This is useful since the directories need to exist for their -## permissions to be preserved (otherwise the docker daemon would create them -## as root). Doing this is the only way to have volume directories which are -## owned by a user ID determined during container runtime, usually as the -## logged in user so that you can share files between the user and the docker -## container without running into permission problems. +THIS_DIR=$(dirname $0) -if [ -z $1 ]; then - echo "Usage: ${0} docker-compose.yml" - echo "This script creates the directories listed as volumes in the docker-compose config file passed as argument." - exit 1; -fi +mkdir -p \ + ${THIS_DIR}/data/mine/dump \ + ${THIS_DIR}/data/mine/configs \ + ${THIS_DIR}/data/mine/packages \ + ${THIS_DIR}/data/mine/intermine \ + ${THIS_DIR}/data/postgres \ + ${THIS_DIR}/data/solr -# awk is the simplest way to parse a YAML without introducing any dependencies. -awk 'BEGIN {FS="\n"} { - for (i=1; i<=NF; i++) { - if ($i ~ /^ *volumes:/) { - reading=1 - } else if (reading) { - if ($i ~ /^ *-/) { - start=index($i, "-")+2 - count=index($i, ":")-start - print substr($i, start, count) - } else if ($i !~ /^ *#/) { - reading=0 - } - } - } -}' $1 | xargs mkdir -p +sudo chown -R 8983:8983 ${THIS_DIR}/data/solr + +echo "Don't forget to also mkdir ${THIS_DIR}/data/mine/" diff --git a/postgres/docker-healthcheck b/postgres/docker-healthcheck index 251c559..e086897 100755 --- a/postgres/docker-healthcheck +++ b/postgres/docker-healthcheck @@ -1,7 +1,7 @@ #!/bin/bash set -eo pipefail -host="${PGHOST:-intermine-postgres}" +host="${PGHOST:-localhost}" port="${PGPORT:-5432}" user="${PSQL_USER:-postgres}" db="mine"