PLAT-200 jsr config importing

.env.dev

@@ -51,8 +51,10 @@ LS_JAVA_OPTS=-Xmx2g -Xms2g
 
 JS_REPORT_INSTANCES=1
 JS_REPORT=dev_password_only
+JS_REPORT_USERNAME=admin
 JS_REPORT_SECRET=dev_secret_only
 JS_REPORT_CPU_LIMIT=0.3
 JS_REPORT_CPU_RESERVE=0.15
 JS_REPORT_MEMORY_LIMIT=6G
 JS_REPORT_MEMORY_RESERVE=2G
+JS_REPORT_SSL=true

dashboard-visualiser-jsreport/docker-compose.await-helper.yml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  await-helper:
+    image: jembi/await-helper:1.0.0
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+    command: 'http://dashboard-visualiser-jsreport:5488/'

dashboard-visualiser-jsreport/importer/docker-compose.config.yml

@@ -0,0 +1,28 @@
+version: '3.9'
+
+services:
+  jsreport-config-importer:
+    # TODO: Update image to use published version of jembi/config-importer
+    image: jembi/api-config-importer:temp
+    environment:
+      SERVICE_NAME: 'dashboard-visualiser-jsreport'
+      SERVICE_API_PORT: 5488
+      API_USERNAME: ${JS_REPORT_USERNAME:-admin}
+      API_PASSWORD: ${JS_REPORT:-dev_password_only}
+      JS_REPORT_SSL: ${JS_REPORT_SSL:-true}
+      API_PATH: '/api/import'
+      BODY_DATA_FORMAT: 'multipart/form-data'
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+    configs:
+      - source: JSR-export.jsrexport
+        target: /export.jsrexport
+
+configs:
+  JSR-export.jsrexport:
+    file: ./export.jsrexport
+    name: JSR-export.jsrexport-${JSR_export_jsrexport_DIGEST:?err}
+    labels:
+      name: jsreport

dashboard-visualiser-jsreport/swarm.sh

@@ -1,28 +1,85 @@
 #!/bin/bash
 
+Action=$1
+Mode=$2
+
 STATEFUL_NODES=${STATEFUL_NODES:-"cluster"}
 
-ComposeFilePath=$(
+COMPOSE_FILE_PATH=$(
   cd "$(dirname "${BASH_SOURCE[0]}")" || exit
   pwd -P
 )
 
-if [[ "$2" == "dev" ]]; then
+ROOT_PATH="${COMPOSE_FILE_PATH}/.."
+. "${ROOT_PATH}/utils/config-utils.sh"
+
+if [[ "$Mode" == "dev" ]]; then
   printf "\nRunning JS Reports package in DEV mode\n"
-  JsReportDevComposeParam="-c ${ComposeFilePath}/docker-compose.dev.yml"
+  JsReportDevComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.dev.yml"
 else
   printf "\nRunning JS Reports package in PROD mode\n"
   JsReportDevComposeParam=""
 fi
 
-if [[ "$1" == "init" ]]; then
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml $JsReportDevComposeParam instant
-elif [[ "$1" == "up" ]]; then
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml $JsReportDevComposeParam instant
-elif [[ "$1" == "down" ]]; then
+AwaitJsrRunning() {
+  local startTime=$(date +%s)
+  until [[ $(docker service ls -f name=instant_dashboard-visualiser-jsreport --format "{{.Replicas}}") == *"${JS_REPORT_INSTANCES}/${JS_REPORT_INSTANCES}"* ]]; do
+    config::timeout_check $startTime "dashboard-visualiser-jsreport to start"
+    sleep 1
+  done
+
+  local awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
+  until [[ $awaitHelperState == *"Complete"* ]]; do
+    config::timeout_check $startTime "dashboard-visualiser-jsreport status check"
+    sleep 1
+
+    awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
+    if [[ $awaitHelperState == *"Failed"* ]] || [[ $awaitHelperState == *"Rejected"* ]]; then
+      echo "Fatal: Received error when trying to verify state of dashboard-visualiser-jsreport. Error:
+       $(docker service ps instant_await-helper --no-trunc --format \"{{.Error}}\")"
+      exit 1
+    fi
+  done
+
+  docker service rm instant_await-helper
+}
+
+RemoveConfigImporter() {
+  local complete="false"
+  local startTime=$(date +%s)
+  local configImporterState=$(docker service ps instant_jsreport-config-importer --format "{{.CurrentState}}")
+  until [[ $configImporterState == *"Complete"* ]]; do
+    config::timeout_check $startTime "jsreport-config-importer to run"
+    sleep 1
+
+    configImporterState=$(docker service ps instant_jsreport-config-importer --format "{{.CurrentState}}")
+    if [[ $configImporterState == *"Failed"* ]] || [[ $configImporterState == *"Rejected"* ]]; then
+      echo "Fatal: JS Reports config importer failed with error:
+       $(docker service ps instant_jsreport-config-importer --no-trunc --format \"{{.Error}}\")"
+      exit 1
+    fi
+  done
+
+  docker service rm instant_jsreport-config-importer
+}
+
+if [[ "$Action" == "init" ]] || [[ "$Action" == "up" ]]; then
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml $JsReportDevComposeParam instant
+
+  docker stack deploy -c "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml instant
+
+  echo "Verifying JS Reports service status"
+  AwaitJsrRunning
+
+  config::set_config_digests "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml instant
+
+  RemoveConfigImporter
+  config::remove_stale_service_configs "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml "jsreport"
+elif [[ "$Action" == "down" ]]; then
   docker service scale instant_dashboard-visualiser-jsreport=0
-elif [[ "$1" == "destroy" ]]; then
-  docker service rm instant_dashboard-visualiser-jsreport
+elif [[ "$Action" == "destroy" ]]; then
+  docker service rm instant_dashboard-visualiser-jsreport instant_jsreport-config-importer instant_await-helper
 else
   echo "Valid options are: init, up, down, or destroy"
 fi

interoperability-layer-openhim/initiateReplicaSet.sh

@@ -2,6 +2,14 @@
 
 echo 'Initiating the mongo replica set'
 
+COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+)
+
+ROOT_PATH="${COMPOSE_FILE_PATH}/.."
+. "${ROOT_PATH}/utils/config-utils.sh"
+
 MONGO_SET_COUNT=${MONGO_SET_COUNT:-3}
 Config='{"_id":"mongo-set","members":['
 Priority="1"
@@ -18,14 +26,7 @@ echo 'Waiting to ensure all the mongo instances for the replica set are up and r
 RunningInstanceCount=0
 StartTime=$(date +%s)
 until [[ $RunningInstanceCount -eq $MONGO_SET_COUNT ]]; do
-    TimeDiff=$(($(date +%s) - $StartTime))
-    if [[ $TimeDiff -ge 60 ]] && [[ $TimeDiff -lt 61 ]]; then
-        echo "Warning: Waited 1 minute for mongo set to start. This is taking longer than it should..."
-    elif [[ $TimeDiff -ge 120 ]]; then
-        echo "Fatal: Waited 2 minutes for mongo set to start. Exiting..."
-        exit 1
-    fi
-
+    config::timeout_check $StartTime "mongo replica set to run"
     sleep 1
 
     RunningInstanceCount=0

interoperability-layer-openhim/swarm.sh

@@ -6,21 +6,24 @@ OPENHIM_MEDIATOR_API_PORT=${OPENHIM_MEDIATOR_API_PORT:-"8080"}
 OPENHIM_CORE_INSTANCES=${OPENHIM_CORE_INSTANCES:-1}
 MONGO_SET_COUNT=${MONGO_SET_COUNT:-3}
 
-ComposeFilePath=$(
+COMPOSE_FILE_PATH=$(
   cd "$(dirname "${BASH_SOURCE[0]}")" || exit
   pwd -P
 )
 
+ROOT_PATH="${COMPOSE_FILE_PATH}/.."
+. "${ROOT_PATH}/utils/config-utils.sh"
+
 VerifyCore() {
   local startTime=$(date +%s)
   until [[ $(docker service ls -f name=instant_openhim-core --format "{{.Replicas}}") == *"$OPENHIM_CORE_INSTANCES/$OPENHIM_CORE_INSTANCES"* ]]; do
-    TimeoutCheck $startTime "openhim-core to start"
+    config::timeout_check $startTime "openhim-core to start"
     sleep 1
   done
 
   local awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
   until [[ $awaitHelperState == *"Complete"* ]]; do
-    TimeoutCheck $startTime "openhim-core heartbeat check"
+    config::timeout_check $startTime "openhim-core heartbeat check"
     sleep 1
 
     awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
@@ -39,7 +42,7 @@ RemoveConfigImporter() {
   local startTime=$(date +%s)
   local configImporterState=$(docker service ps instant_interoperability-layer-openhim-config-importer --format "{{.CurrentState}}")
   until [[ $configImporterState == *"Complete"* ]]; do
-    TimeoutCheck $startTime "interoperability-layer-openhim-config-importer to run"
+    config::timeout_check $startTime "interoperability-layer-openhim-config-importer to run"
     sleep 1
 
     configImporterState=$(docker service ps instant_interoperability-layer-openhim-config-importer --format "{{.CurrentState}}")
@@ -53,24 +56,12 @@ RemoveConfigImporter() {
   docker service rm instant_interoperability-layer-openhim-config-importer
 }
 
-TimeoutCheck() {
-  local startTime=$(($1))
-  local message=$2
-  local timeDiff=$(($(date +%s) - $startTime))
-  if [[ $timeDiff -ge 60 ]] && [[ $timeDiff -lt 61 ]]; then
-    echo "Warning: Waited 1 minute for $message. This is taking longer than it should..."
-  elif [[ $timeDiff -ge 120 ]]; then
-    echo "Fatal: Waited 2 minutes for $message. Exiting..."
-    exit 1
-  fi
-}
-
 VerifyMongos() {
   echo 'Waiting to ensure all the mongo instances for the replica set are up and running'
   local runningInstanceCount=0
   local startTime=$(date +%s)
   until [[ $runningInstanceCount -eq $MONGO_SET_COUNT ]]; do
-    TimeoutCheck $startTime "mongo set to start"
+    config::timeout_check $startTime "mongo set to start"
     sleep 1
 
     runningInstanceCount=0
@@ -84,55 +75,55 @@ VerifyMongos() {
 
 if [[ $STATEFUL_NODES == "cluster" ]]; then
   printf "\nRunning Interoperability Layer OpenHIM package in Cluster node mode\n"
-  MongoClusterComposeParam="-c ${ComposeFilePath}/docker-compose-mongo.cluster.yml"
+  MongoClusterComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose-mongo.cluster.yml"
 else
   printf "\nRunning Interoperability Layer OpenHIM package in Single node mode\n"
   MongoClusterComposeParam=""
 fi
 
 if [[ "$2" == "dev" ]]; then
   printf "\nRunning Interoperability Layer OpenHIM package in DEV mode\n"
-  MongoDevComposeParam="-c ${ComposeFilePath}/docker-compose-mongo.dev.yml"
-  OpenhimDevComposeParam="-c ${ComposeFilePath}/docker-compose.dev.yml"
+  MongoDevComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose-mongo.dev.yml"
+  OpenhimDevComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.dev.yml"
 else
   printf "\nRunning Interoperability Layer OpenHIM package in PROD mode\n"
   MongoDevComposeParam=""
   OpenhimDevComposeParam=""
 fi
 
 if [[ "$1" == "init" ]]; then
-  docker stack deploy -c "$ComposeFilePath"/docker-compose-mongo.yml $MongoClusterComposeParam $MongoDevComposeParam instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose-mongo.yml $MongoClusterComposeParam $MongoDevComposeParam instant
 
   # Set up the replica set
-  "$ComposeFilePath"/initiateReplicaSet.sh
-  if [[ $? -eq 1 ]]; then
+  "$COMPOSE_FILE_PATH"/initiateReplicaSet.sh
+  if [[ $? -ne 0 ]]; then
     echo "Fatal: Initate Mongo replica set failed."
     exit 1
   fi
 
   # Set host in OpenHIM console config
   sed -i "s/localhost/$OPENHIM_CORE_MEDIATOR_HOSTNAME/g; s/8080/$OPENHIM_MEDIATOR_API_PORT/g" /instant/interoperability-layer-openhim/importer/volume/default.json
 
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml -c "$ComposeFilePath"/docker-compose.stack-0.yml $OpenhimDevComposeParam instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml -c "$COMPOSE_FILE_PATH"/docker-compose.stack-0.yml $OpenhimDevComposeParam instant
 
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.await-helper.yml instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.await-helper.yml instant
 
   echo "Waiting to give OpenHIM Core time to start up before OpenHIM Console run"
   VerifyCore
 
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml -c "$ComposeFilePath"/docker-compose.stack-1.yml $OpenhimDevComposeParam instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml -c "$COMPOSE_FILE_PATH"/docker-compose.stack-1.yml $OpenhimDevComposeParam instant
 
-  docker stack deploy -c "$ComposeFilePath"/importer/docker-compose.config.yml instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml instant
 
   echo "Waiting to give core config importer time to run before cleaning up service"
   RemoveConfigImporter
 
   # Sleep to ensure config importer is removed
   sleep 5
 elif [[ "$1" == "up" ]]; then
-  docker stack deploy -c "$ComposeFilePath"/docker-compose-mongo.yml $MongoClusterComposeParam $MongoDevComposeParam instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose-mongo.yml $MongoClusterComposeParam $MongoDevComposeParam instant
   VerifyMongos
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml -c "$ComposeFilePath"/docker-compose.stack-1.yml $OpenhimDevComposeParam instant
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml -c "$COMPOSE_FILE_PATH"/docker-compose.stack-1.yml $OpenhimDevComposeParam instant
 elif [[ "$1" == "down" ]]; then
   docker service scale instant_openhim-core=0 instant_openhim-console=0 instant_mongo-1=0 instant_mongo-2=0 instant_mongo-3=0
 elif [[ "$1" == "destroy" ]]; then

utils/config-utils.sh

@@ -109,3 +109,26 @@ config::remove_stale_service_configs() {
 
     docker config rm "${configsToRemove[@]}"
 }
+
+# A function that exists in a loop to see how long that loop has run for, providing a warning
+# at the time specified in argument $3, and exits with code 124 after the time specified in argument $4.
+#
+# Arguments:
+# $1 : start time of the timeout check
+# $2 : a message containing reference to the loop that timed out
+# $3 : elapsed time to issue running-for-longer-than-expected warning (in seconds), default is 60 seconds
+# $4 : timeout time in seconds, default is 300 seconds
+config::timeout_check() {
+    local startTime=$(($1))
+    local message=$2
+    local exitTime="${3:-300}"
+    local warningTime="${4:-60}"
+
+    local timeDiff=$(($(date +%s) - $startTime))
+    if [[ $timeDiff -ge $warningTime ]] && [[ $timeDiff -lt $(($warningTime + 1)) ]]; then
+        echo "Warning: Waited $warningTime seconds for $message. This is taking longer than it should..."
+    elif [[ $timeDiff -ge $exitTime ]]; then
+        echo "Fatal: Waited $exitTime seconds for $message. Exiting..."
+        exit 124
+    fi
+}