From 7d2a7badefab69046e4e4cfa6f7fd70310ce1730 Mon Sep 17 00:00:00 2001
From: Nicholas Long <nicholas.long@nrel.gov>
Date: Tue, 17 Jul 2018 12:06:16 -0600
Subject: [PATCH] add wait-for-it to ensure processes start  in the correct
 order

---
 Dockerfile                               |  41 +++---
 docker-compose.deploy.yml                |   5 +-
 docker-compose.local.yml                 |   5 +-
 docker-compose.yml                       |   5 +-
 docker/deployment/docker-compose.aws.yml |   5 +-
 docker/development/docker-compose.yml    |   1 +
 docker/server/rails-entrypoint.sh        |   7 +-
 docker/server/run-server-tests.sh        |   6 +
 docker/server/start-server-dev.sh        |   6 +
 docker/server/start-server.sh            |   8 +
 docker/server/start-web-background.sh    |   9 ++
 docker/server/start-workers.sh           |   9 ++
 docker/server/wait-for-it.sh             | 178 +++++++++++++++++++++++
 13 files changed, 254 insertions(+), 31 deletions(-)
 create mode 100644 docker/server/start-web-background.sh
 create mode 100644 docker/server/start-workers.sh
 create mode 100755 docker/server/wait-for-it.sh

diff --git a/Dockerfile b/Dockerfile
index 63e8557e5..ddf5acdc3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,7 +9,8 @@ MAINTAINER Nicholas Long nicholas.long@nrel.gov
 
 RUN ruby -r openstudio -e "require 'openstudio'; puts OpenStudio.openStudioLongVersion"
 
-# Install required libaries
+# Install required libaries.
+#   realpath - needed for wait-for-it
 RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && \
     echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | \
     sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list && \
@@ -47,6 +48,7 @@ RUN sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 &
         libsm-dev \
         mongodb-org-tools \
         procps \
+        realpath \
         tar \
         unzip \
         wget \
@@ -106,34 +108,28 @@ RUN bundle install --jobs=3 --retry=3
 ADD /docker/server/ipvs-keepalive.conf /etc/sysctl.d/ipvs-keepalive.conf
 RUN sudo sysctl --system
 
-RUN chmod 775 /opt/openstudio/server/log
-RUN chmod 666 /opt/openstudio/server/log/*.log
-
-ADD /docker/server/start-server.sh /usr/local/bin/start-server
-ADD /docker/server/run-server-tests.sh /usr/local/bin/run-server-tests
-ADD /docker/server/memfix-controller.rb /usr/local/lib/memfix-controller.rb
-ADD /docker/server/memfix.rb /usr/local/lib/memfix.rb
+# Add in scripts for running server. This includes the wait-for-it scripts to ensure other processes (mongo, redis) have
+# started before starting the main process.
+COPY /docker/server/wait-for-it.sh /usr/local/bin/wait-for-it
+COPY /docker/server/start-server.sh /usr/local/bin/start-server
+COPY /docker/server/run-server-tests.sh /usr/local/bin/run-server-tests
+COPY /docker/server/rails-entrypoint.sh /usr/local/bin/rails-entrypoint
+COPY /docker/server/start-web-background.sh /usr/local/bin/start-web-background
+COPY /docker/server/start-workers.sh /usr/local/bin/start-workers
+COPY /docker/server/memfix-controller.rb /usr/local/lib/memfix-controller.rb
+COPY /docker/server/memfix.rb /usr/local/lib/memfix.rb
+RUN chmod 755 /usr/local/bin/wait-for-it
 RUN chmod +x /usr/local/bin/start-server
 RUN chmod +x /usr/local/bin/run-server-tests
+RUN chmod 755 /usr/local/bin/rails-entrypoint
+RUN chmod 755 /usr/local/bin/start-web-background
+RUN chmod 755 /usr/local/bin/start-workers
 RUN chmod +x /usr/local/lib/memfix-controller.rb
 RUN chmod +x /usr/local/lib/memfix.rb
 
 # set the permissions for windows users
 RUN chmod +x /opt/openstudio/server/bin/*
 
-# permissions on where server assets (e.g. paperclip, data points, R images, etc) are stored
-RUN mkdir -p /mnt/openstudio/server/R && chmod 777 /mnt/openstudio/server/R
-RUN mkdir -p /mnt/openstudio/server/assets && chmod 777 /mnt/openstudio/server/assets
-#RUN mkdir -p /mnt/openstudio/server/assets/data_points && chmod 777 /mnt/openstudio/server/assets/data_points
-RUN mkdir -p /mnt/openstudio/server/assets/variables && chmod 777 /mnt/openstudio/server/assets/variables
-RUN mkdir -p /opt/openstudio/server/tmp && chmod 777 /opt/openstudio/server/tmp
-
-# Test adding the git repo to the container for coveralls
-# The #TEST# will be removed in the travis test script to be run in the test container
-#TEST#ADD .git /opt/openstudio/.git
-
-ADD /docker/server/rails-entrypoint.sh /usr/local/bin/rails-entrypoint
-RUN chmod 755 /usr/local/bin/rails-entrypoint
 ENTRYPOINT ["rails-entrypoint"]
 
 CMD ["/usr/local/bin/start-server"]
@@ -162,3 +158,6 @@ RUN echo "Running in testing environment - Installing Firefox and Gecko Driver"
     rm geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \
     chmod +x geckodriver;
 
+# Test adding the git repo to the container for coveralls
+# The #TEST# will be removed in the travis test script to be run in the test container
+#TEST#COPY .git /opt/openstudio/.git
diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml
index 57e5ee6a0..c19f0f2f8 100644
--- a/docker-compose.deploy.yml
+++ b/docker-compose.deploy.yml
@@ -35,6 +35,7 @@ services:
       resources:
         reservations:
           cpus: "1"
+    command: /usr/local/bin/start-server
   web-background:
     image: nrel/openstudio-server:latest
     environment:
@@ -42,7 +43,6 @@ services:
       - QUEUES=background,analyses
     volumes:
       - osdata:/mnt/openstudio
-    command: bundle exec rake environment resque:work
     depends_on:
       - db
       - queue
@@ -54,9 +54,9 @@ services:
       resources:
         reservations:
           cpus: "1"
+    command: /usr/local/bin/start-web-background
   worker:
     image: nrel/openstudio-server:latest
-    command: bundle exec rake environment resque:workers
     environment:
       - QUEUES=simulations
       - COUNT=1
@@ -71,6 +71,7 @@ services:
       resources:
         reservations:
           cpus: "0.99"
+    command: /usr/local/bin/start-workers
   rserve:
     image: nrel/openstudio-rserve:latest
     volumes:
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 7c4b60a10..7900a8b9b 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -28,6 +28,7 @@ services:
       - "8080:80"
     volumes:
       - osdata:/mnt/openstudio
+    command: /usr/local/bin/start-server
   web-background:
     image: 127.0.0.1:5000/openstudio-server
     build:
@@ -44,7 +45,7 @@ services:
       - web
     volumes:
       - osdata:/mnt/openstudio
-    command: bundle exec rake environment resque:work
+    command: /usr/local/bin/start-web-background
   worker:
     image: 127.0.0.1:5000/openstudio-server
     build:
@@ -61,7 +62,7 @@ services:
       - db
       - queue
       - rserve
-    command: bundle exec rake environment resque:workers
+    command: /usr/local/bin/start-workers
   rserve:
     image: 127.0.0.1:5000/openstudio-rserve
     build: ./docker/R
diff --git a/docker-compose.yml b/docker-compose.yml
index fc26bb201..c88a0762f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -28,6 +28,7 @@ services:
       - "8080:80"
     volumes:
       - osdata:/mnt/openstudio
+    command: /usr/local/bin/start-server
   web-background:
     image: nrel/openstudio-server:latest
     build:
@@ -44,7 +45,7 @@ services:
       - web
     volumes:
       - osdata:/mnt/openstudio
-    command: bundle exec rake environment resque:work
+    command: /usr/local/bin/start-web-background
   worker:
     image: nrel/openstudio-server:latest
     build:
@@ -61,7 +62,7 @@ services:
       - db
       - queue
       - rserve
-    command: bundle exec rake environment resque:workers
+    command: /usr/local/bin/start-workers
   rserve:
     image: nrel/openstudio-rserve:latest
     build: ./docker/R
diff --git a/docker/deployment/docker-compose.aws.yml b/docker/deployment/docker-compose.aws.yml
index de085adab..a883baeda 100644
--- a/docker/deployment/docker-compose.aws.yml
+++ b/docker/deployment/docker-compose.aws.yml
@@ -46,6 +46,7 @@ services:
       resources:
         reservations:
           cpus: "AWS_WEB_CORES"
+    command: /usr/local/bin/start-server
   web-background:
     image: localhost:5000/openstudio-server
     environment:
@@ -53,7 +54,6 @@ services:
       - QUEUES=background,analyses
     volumes:
       - osdata:/mnt/openstudio
-    command: bundle exec rake environment resque:work
     depends_on:
       - db
       - web
@@ -65,12 +65,12 @@ services:
       resources:
         reservations:
           cpus: "1"
+    command: /usr/local/bin/start-web-background
   worker:
     image: localhost:5000/openstudio-server
     environment:
       - QUEUES=simulations
       - COUNT=1
-    command: bundle exec rake environment resque:workers
     volumes:
       - /mnt/openstudio
     depends_on:
@@ -83,6 +83,7 @@ services:
       resources:
         reservations:
           cpus: "1"
+    command: /usr/local/bin/start-workers
   rserve:
     image: localhost:5000/openstudio-rserve
     volumes:
diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml
index a5f6c396b..4c927c16e 100644
--- a/docker/development/docker-compose.yml
+++ b/docker/development/docker-compose.yml
@@ -38,6 +38,7 @@ services:
     volumes:
       - osdata:/mnt/openstudio
       - ../..:/opt/openstudio/
+    # TODO: This is not using the wait-for-it script, not sure if we still want this file.
     command: ../server/bin/delayed_job -i server --queues=analyses,background run
   worker:
     build:
diff --git a/docker/server/rails-entrypoint.sh b/docker/server/rails-entrypoint.sh
index 387fd5647..b6fbc3f9d 100644
--- a/docker/server/rails-entrypoint.sh
+++ b/docker/server/rails-entrypoint.sh
@@ -9,14 +9,17 @@ mkdir -p /opt/openstudio/server/tmp && chmod 777 /opt/openstudio/server/tmp
 mkdir -p /mnt/coredumps/ && chmod 777 /mnt/coredumps/
 sleep 1
 
-echo 'Defaulting required variables which are not present'
+echo 'Setting permissions on logs'
+chmod 775 /opt/openstudio/server/log
+chmod 666 /opt/openstudio/server/log/*.log
 sleep 1
+
+echo 'Defaulting required variables which are not present'
 [ -z "$MAX_REQUESTS" ] && export MAX_REQUESTS=50;
 [ -z "$MAX_POOL" ] && export MAX_POOL=16;
 sleep 1
 
 echo 'Configuring nginx'
-sleep 1
 { rm /opt/nginx/conf/nginx.conf && awk  -v MAX_REQUESTS=$MAX_REQUESTS '{gsub("MAX_REQUESTS_SUB", MAX_REQUESTS, $0); print}' > /opt/nginx/conf/nginx.conf; } < /opt/nginx/conf/nginx.conf
 { rm /opt/nginx/conf/nginx.conf && awk  -v MAX_POOL=$MAX_POOL '{gsub("MAX_POOL_SUB", MAX_POOL, $0); print}' > /opt/nginx/conf/nginx.conf; } < /opt/nginx/conf/nginx.conf
 sleep 1
diff --git a/docker/server/run-server-tests.sh b/docker/server/run-server-tests.sh
index 62db94b94..c03bb27a8 100755
--- a/docker/server/run-server-tests.sh
+++ b/docker/server/run-server-tests.sh
@@ -11,6 +11,12 @@ do
   sleep 1s
 done
 
+echo "Waiting for Mongo to start"
+/usr/local/bin/wait-for-it --strict db:27017
+
+echo "Waiting for Redis to start"
+/usr/local/bin/wait-for-it --strict queue:6379
+
 # Always create new indexes in case the models have changed
 cd /opt/openstudio/server && bundle exec rspec; (( exit_status = exit_status || $? ))
 cd /opt/openstudio/server && bundle exec rake rubocop:run; (( exit_status = exit_status || $? ))
diff --git a/docker/server/start-server-dev.sh b/docker/server/start-server-dev.sh
index 358f1ccd9..f14ff9097 100755
--- a/docker/server/start-server-dev.sh
+++ b/docker/server/start-server-dev.sh
@@ -1,5 +1,11 @@
 #!/usr/bin/env bash
 
+echo "Waiting for Mongo to start"
+/usr/local/bin/wait-for-it --strict db:27017
+
+echo "Waiting for Redis to start"
+/usr/local/bin/wait-for-it --strict queue:6379
+
 # Always create new indexes in case the models have changed
 cd /opt/openstudio/server && bundle exec rake db:mongoid:create_indexes
 
diff --git a/docker/server/start-server.sh b/docker/server/start-server.sh
index ef8d9b569..2124adc5d 100755
--- a/docker/server/start-server.sh
+++ b/docker/server/start-server.sh
@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 
+echo "Waiting for Mongo to start"
+/usr/local/bin/wait-for-it --strict db:27017
+
+echo "Waiting for Redis to start"
+/usr/local/bin/wait-for-it --strict queue:6379
+
 # Always create new indexes in case the models have changed
 cd /opt/openstudio/server && bundle exec rake db:mongoid:create_indexes
 
+# The memfix-controller seems to have causes some issues with NRCan, need to revisit.
+# https://github.com/NREL/OpenStudio-server/issues/348
 ruby /usr/local/lib/memfix-controller.rb start
 
 /opt/nginx/sbin/nginx
diff --git a/docker/server/start-web-background.sh b/docker/server/start-web-background.sh
new file mode 100644
index 000000000..335bbc295
--- /dev/null
+++ b/docker/server/start-web-background.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "Waiting for Mongo to start"
+/usr/local/bin/wait-for-it --strict db:27017
+
+echo "Waiting for Redis to start"
+/usr/local/bin/wait-for-it --strict queue:6379
+
+cd /opt/openstudio/server && bundle exec rake environment resque:work
diff --git a/docker/server/start-workers.sh b/docker/server/start-workers.sh
new file mode 100644
index 000000000..374cb28e5
--- /dev/null
+++ b/docker/server/start-workers.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "Waiting for Mongo to start"
+/usr/local/bin/wait-for-it --strict db:27017
+
+echo "Waiting for Redis to start"
+/usr/local/bin/wait-for-it --strict queue:6379
+
+cd /opt/openstudio/server && bundle exec rake environment resque:workers
diff --git a/docker/server/wait-for-it.sh b/docker/server/wait-for-it.sh
new file mode 100755
index 000000000..b9f647531
--- /dev/null
+++ b/docker/server/wait-for-it.sh
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+# From: https://github.com/vishnubob/wait-for-it
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $TIMEOUT -gt 0 ]]; then
+        echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+    else
+        echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+    fi
+    start_ts=$(date +%s)
+    while :
+    do
+        if [[ $ISBUSY -eq 1 ]]; then
+            nc -z $HOST $PORT
+            result=$?
+        else
+            (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+            result=$?
+        fi
+        if [[ $result -eq 0 ]]; then
+            end_ts=$(date +%s)
+            echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $QUIET -eq 1 ]]; then
+        timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    else
+        timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    fi
+    PID=$!
+    trap "kill -INT -$PID" INT
+    wait $PID
+    RESULT=$?
+    if [[ $RESULT -ne 0 ]]; then
+        echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
+    fi
+    return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        hostport=(${1//:/ })
+        HOST=${hostport[0]}
+        PORT=${hostport[1]}
+        shift 1
+        ;;
+        --child)
+        CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        STRICT=1
+        shift 1
+        ;;
+        -h)
+        HOST="$2"
+        if [[ $HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        PORT="$2"
+        if [[ $PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        TIMEOUT="$2"
+        if [[ $TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        CLI="$@"
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+# check to see if timeout is from busybox?
+# check to see if timeout is from busybox?
+TIMEOUT_PATH=$(realpath $(which timeout))
+if [[ $TIMEOUT_PATH =~ "busybox" ]]; then
+        ISBUSY=1
+        BUSYTIMEFLAG="-t"
+else
+        ISBUSY=0
+        BUSYTIMEFLAG=""
+fi
+
+if [[ $CHILD -gt 0 ]]; then
+    wait_for
+    RESULT=$?
+    exit $RESULT
+else
+    if [[ $TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        RESULT=$?
+    else
+        wait_for
+        RESULT=$?
+    fi
+fi
+
+if [[ $CLI != "" ]]; then
+    if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+        echoerr "$cmdname: strict mode, refusing to execute subprocess"
+        exit $RESULT
+    fi
+    exec $CLI
+else
+    exit $RESULT
+fi