Merge pull request #44 from jembi/PLAT-200-jsr-config-importing

PLAT-200 jsr config importing

.env.dev

@@ -51,8 +51,11 @@ LS_JAVA_OPTS=-Xmx2g -Xms2g
 
 JS_REPORT_INSTANCES=1
 JS_REPORT=dev_password_only
+JS_REPORT_USERNAME=admin
 JS_REPORT_SECRET=dev_secret_only
 JS_REPORT_CPU_LIMIT=0.3
 JS_REPORT_CPU_RESERVE=0.15
 JS_REPORT_MEMORY_LIMIT=6G
 JS_REPORT_MEMORY_RESERVE=2G
+JS_REPORT_SSL=true
+JS_REPORT_CONFIG_FILE=export.jsrexport

dashboard-visualiser-jsreport/docker-compose.await-helper.yml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  await-helper:
+    image: jembi/await-helper:1.0.0
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+    command: 'http://dashboard-visualiser-jsreport:5488/'

dashboard-visualiser-jsreport/docker-compose.yml

@@ -9,10 +9,10 @@ services:
       replicas: ${JS_REPORT_INSTANCES:-1}
       resources:
         limits:
-          cpus: ${JS_REPORT_CPU_LIMIT:-0.25}
-          memory: ${JS_REPORT_MEMORY_LIMIT:-4G}
+          cpus: ${JS_REPORT_CPU_LIMIT:-0.15}
+          memory: ${JS_REPORT_MEMORY_LIMIT:-3G}
         reservations:
-          cpus: ${JS_REPORT_CPU_RESERVE:-0.1}
+          cpus: ${JS_REPORT_CPU_RESERVE:-0.05}
           memory: ${JS_REPORT_MEMORY_RESERVE:-1G}
     labels:
       co.elastic.metrics/module: 'docker'

dashboard-visualiser-jsreport/importer/docker-compose.config.yml

@@ -0,0 +1,29 @@
+version: '3.9'
+
+services:
+  jsreport-config-importer:
+    # TODO: Update image to use published version of jembi/config-importer
+    image: jembi/api-config-importer:latest
+    environment:
+      SERVICE_NAME: 'dashboard-visualiser-jsreport'
+      SERVICE_API_PORT: 5488
+      API_USERNAME: ${JS_REPORT_USERNAME:-admin}
+      API_PASSWORD: ${JS_REPORT:-dev_password_only}
+      SSL: ${JS_REPORT_SSL:-true}
+      API_PATH: '/api/import'
+      MIME_TYPE: 'multipart/form-data'
+      CONFIG_FILE: ${JS_REPORT_CONFIG_FILE:-export.jsrexport}
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+    configs:
+      - source: JSR-export.jsrexport
+        target: /export.jsrexport
+
+configs:
+  JSR-export.jsrexport:
+    file: ./export.jsrexport
+    name: JSR-export.jsrexport-${JSR_export_jsrexport_DIGEST:?err}
+    labels:
+      name: jsreport

dashboard-visualiser-jsreport/package-metadata.json

@@ -4,13 +4,18 @@
   "description": "A dashboard to interpret data from ElasticSearch data store",
   "type": "infrastructure",
   "version": "v0.0.1",
-  "dependencies": [],
+  "dependencies": ["analytics-datastore-elastic-search"],
   "environmentVariables": {
-    "allowLocalFilesAccess": "true",
-    "extensions_authentication_cookieSession_secret": "${JS_REPORT_SECRET:-dev_secret_only}",
-    "extensions_authentication_admin_username": "admin",
-    "extensions_authentication_admin_password": "${JS_REPORT:-dev_password_only}",
-    "ES_PASSWORD": "${ES_ELASTIC:-dev_password_only}",
-    "extensions_fs-store_dataDirectory": "jsreport/data"
+    "JS_REPORT_SECRET": "dev_secret_only",
+    "JS_REPORT": "dev_password_only",
+    "ES_PASSWORD": "dev_password_only",
+    "API_USERNAME": "admin",
+    "API_PASSWORD": "dev_password_only",
+    "JS_REPORT_SSL": "true",
+    "JS_REPORT_CONFIG_FILE": "export.jsrexport",
+    "JS_REPORT_CPU_LIMIT": "0.15",
+    "JS_REPORT_MEMORY_LIMIT": "3G",
+    "JS_REPORT_CPU_RESERVE": "0.05",
+    "JS_REPORT_MEMORY_RESERVE": "1G"
   }
 }

dashboard-visualiser-jsreport/swarm.sh

@@ -1,28 +1,85 @@
 #!/bin/bash
 
+Action=$1
+Mode=$2
+
 STATEFUL_NODES=${STATEFUL_NODES:-"cluster"}
 
-ComposeFilePath=$(
+COMPOSE_FILE_PATH=$(
   cd "$(dirname "${BASH_SOURCE[0]}")" || exit
   pwd -P
 )
 
-if [[ "$2" == "dev" ]]; then
+ROOT_PATH="${COMPOSE_FILE_PATH}/.."
+. "${ROOT_PATH}/utils/config-utils.sh"
+
+if [[ "$Mode" == "dev" ]]; then
   printf "\nRunning JS Reports package in DEV mode\n"
-  JsReportDevComposeParam="-c ${ComposeFilePath}/docker-compose.dev.yml"
+  JsReportDevComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.dev.yml"
 else
   printf "\nRunning JS Reports package in PROD mode\n"
   JsReportDevComposeParam=""
 fi
 
-if [[ "$1" == "init" ]]; then
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml $JsReportDevComposeParam instant
-elif [[ "$1" == "up" ]]; then
-  docker stack deploy -c "$ComposeFilePath"/docker-compose.yml $JsReportDevComposeParam instant
-elif [[ "$1" == "down" ]]; then
+AwaitJsrRunning() {
+  local startTime=$(date +%s)
+  until [[ $(docker service ls -f name=instant_dashboard-visualiser-jsreport --format "{{.Replicas}}") == *"${JS_REPORT_INSTANCES}/${JS_REPORT_INSTANCES}"* ]]; do
+    config::timeout_check $startTime "dashboard-visualiser-jsreport to start"
+    sleep 1
+  done
+
+  local awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
+  until [[ $awaitHelperState == *"Complete"* ]]; do
+    config::timeout_check $startTime "dashboard-visualiser-jsreport status check"
+    sleep 1
+
+    awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
+    if [[ $awaitHelperState == *"Failed"* ]] || [[ $awaitHelperState == *"Rejected"* ]]; then
+      echo "Fatal: Received error when trying to verify state of dashboard-visualiser-jsreport. Error:
+       $(docker service ps instant_await-helper --no-trunc --format \"{{.Error}}\")"
+      exit 1
+    fi
+  done
+
+  docker service rm instant_await-helper
+}
+
+RemoveConfigImporter() {
+  local complete="false"
+  local startTime=$(date +%s)
+  local configImporterState=$(docker service ps instant_jsreport-config-importer --format "{{.CurrentState}}")
+  until [[ $configImporterState == *"Complete"* ]]; do
+    config::timeout_check $startTime "jsreport-config-importer to run"
+    sleep 1
+
+    configImporterState=$(docker service ps instant_jsreport-config-importer --format "{{.CurrentState}}")
+    if [[ $configImporterState == *"Failed"* ]] || [[ $configImporterState == *"Rejected"* ]]; then
+      echo "Fatal: JS Reports config importer failed with error:
+       $(docker service ps instant_jsreport-config-importer --no-trunc --format \"{{.Error}}\")"
+      exit 1
+    fi
+  done
+
+  docker service rm instant_jsreport-config-importer
+}
+
+if [[ "$Action" == "init" ]] || [[ "$Action" == "up" ]]; then
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/docker-compose.yml $JsReportDevComposeParam instant
+
+  docker stack deploy -c "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml instant
+
+  echo "Verifying JS Reports service status"
+  AwaitJsrRunning
+
+  config::set_config_digests "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml
+  docker stack deploy -c "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml instant
+
+  RemoveConfigImporter
+  config::remove_stale_service_configs "$COMPOSE_FILE_PATH"/importer/docker-compose.config.yml "jsreport"
+elif [[ "$Action" == "down" ]]; then
   docker service scale instant_dashboard-visualiser-jsreport=0
-elif [[ "$1" == "destroy" ]]; then
-  docker service rm instant_dashboard-visualiser-jsreport
+elif [[ "$Action" == "destroy" ]]; then
+  docker service rm instant_dashboard-visualiser-jsreport instant_jsreport-config-importer instant_await-helper
 else
   echo "Valid options are: init, up, down, or destroy"
 fi

interoperability-layer-openhim/initiateReplicaSet.sh

@@ -2,6 +2,14 @@
 
 echo 'Initiating the mongo replica set'
 
+COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+)
+
+ROOT_PATH="${COMPOSE_FILE_PATH}/.."
+. "${ROOT_PATH}/utils/config-utils.sh"
+
 MONGO_SET_COUNT=${MONGO_SET_COUNT:-3}
 Config='{"_id":"mongo-set","members":['
 Priority="1"
@@ -18,14 +26,7 @@ echo 'Waiting to ensure all the mongo instances for the replica set are up and r
 RunningInstanceCount=0
 StartTime=$(date +%s)
 until [[ $RunningInstanceCount -eq $MONGO_SET_COUNT ]]; do
-    TimeDiff=$(($(date +%s) - $StartTime))
-    if [[ $TimeDiff -ge 60 ]] && [[ $TimeDiff -lt 61 ]]; then
-        echo "Warning: Waited 1 minute for mongo set to start. This is taking longer than it should..."
-    elif [[ $TimeDiff -ge 120 ]]; then
-        echo "Fatal: Waited 2 minutes for mongo set to start. Exiting..."
-        exit 1
-    fi
-
+    config::timeout_check $StartTime "mongo replica set to run"
     sleep 1
 
     RunningInstanceCount=0

interoperability-layer-openhim/swarm.sh

@@ -12,19 +12,20 @@ COMPOSE_FILE_PATH=$(
 )
 
 # Import libraries
+
 ROOT_PATH="${COMPOSE_FILE_PATH}/.."
 . "${ROOT_PATH}/utils/config-utils.sh"
 
 VerifyCore() {
   local startTime=$(date +%s)
   until [[ $(docker service ls -f name=instant_openhim-core --format "{{.Replicas}}") == *"$OPENHIM_CORE_INSTANCES/$OPENHIM_CORE_INSTANCES"* ]]; do
-    TimeoutCheck $startTime "openhim-core to start"
+    config::timeout_check $startTime "openhim-core to start"
     sleep 1
   done
 
   local awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
   until [[ $awaitHelperState == *"Complete"* ]]; do
-    TimeoutCheck $startTime "openhim-core heartbeat check"
+    config::timeout_check $startTime "openhim-core heartbeat check"
     sleep 1
 
     awaitHelperState=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
@@ -43,7 +44,7 @@ RemoveConfigImporter() {
   local startTime=$(date +%s)
   local configImporterState=$(docker service ps instant_interoperability-layer-openhim-config-importer --format "{{.CurrentState}}")
   until [[ $configImporterState == *"Complete"* ]]; do
-    TimeoutCheck $startTime "interoperability-layer-openhim-config-importer to run"
+    config::timeout_check $startTime "interoperability-layer-openhim-config-importer to run"
     sleep 1
 
     configImporterState=$(docker service ps instant_interoperability-layer-openhim-config-importer --format "{{.CurrentState}}")
@@ -57,24 +58,12 @@ RemoveConfigImporter() {
   docker service rm instant_interoperability-layer-openhim-config-importer
 }
 
-TimeoutCheck() {
-  local startTime=$(($1))
-  local message=$2
-  local timeDiff=$(($(date +%s) - $startTime))
-  if [[ $timeDiff -ge 60 ]] && [[ $timeDiff -lt 61 ]]; then
-    echo "Warning: Waited 1 minute for $message. This is taking longer than it should..."
-  elif [[ $timeDiff -ge 120 ]]; then
-    echo "Fatal: Waited 2 minutes for $message. Exiting..."
-    exit 1
-  fi
-}
-
 VerifyMongos() {
   echo 'Waiting to ensure all the mongo instances for the replica set are up and running'
   local runningInstanceCount=0
   local startTime=$(date +%s)
   until [[ $runningInstanceCount -eq $MONGO_SET_COUNT ]]; do
-    TimeoutCheck $startTime "mongo set to start"
+    config::timeout_check $startTime "mongo set to start"
     sleep 1
 
     runningInstanceCount=0

utils/config-utils.sh

@@ -109,3 +109,26 @@ config::remove_stale_service_configs() {
 
     docker config rm "${configsToRemove[@]}"
 }
+
+# A function that exists in a loop to see how long that loop has run for, providing a warning
+# at the time specified in argument $3, and exits with code 124 after the time specified in argument $4.
+#
+# Arguments:
+# $1 : start time of the timeout check
+# $2 : a message containing reference to the loop that timed out
+# $3 : timeout time in seconds, default is 300 seconds
+# $4 : elapsed time to issue running-for-longer-than-expected warning (in seconds), default is 60 seconds
+config::timeout_check() {
+    local startTime=$(($1))
+    local message=$2
+    local exitTime="${3:-300}"
+    local warningTime="${4:-60}"
+
+    local timeDiff=$(($(date +%s) - $startTime))
+    if [[ $timeDiff -ge $warningTime ]] && [[ $timeDiff -lt $(($warningTime + 1)) ]]; then
+        echo "Warning: Waited $warningTime seconds for $message. This is taking longer than it should..."
+    elif [[ $timeDiff -ge $exitTime ]]; then
+        echo "Fatal: Waited $exitTime seconds for $message. Exiting..."
+        exit 124
+    fi
+}