Wait for agent startup before updating CAPI tags (#145)

* Properly wait for agent startup

* Wait for token as well

* small refactor

* Add back datadog dir when using as argument; harmonize compile and supply

* Fix pid file name

Co-authored-by: Noueman Khalikine <noueman.khalikine@datadoghq.com>

bin/compile

@@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
+cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
@@ -48,6 +49,7 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
 fi
 
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
+chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 chmod +x "${DATADOG_DIR}/trace-agent"

bin/supply

@@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
+cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
@@ -48,6 +49,7 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
 fi
 
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
+chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 chmod +x "${DATADOG_DIR}/trace-agent"

lib/scripts/check_datadog.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022-Present Datadog, Inc.
+
+DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
+
+check_datadog() {
+  while true; do
+    echo "Waiting for agent or dogstatsd process to start"
+    if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
+        echo "Found agent process"
+        if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
+          echo "Found agent token"
+          break
+        else 
+          echo "Agent token not found"
+        fi
+    fi
+
+    if [ -f "${DATADOG_DIR}/run/dogstatsd.pid" ]; then
+        echo "Found dogstatsd process"
+        break
+    fi
+    sleep 1
+  done
+}
+
+main() {
+    check_datadog
+}
+
+main "$@"

lib/scripts/update_agent_config.sh

@@ -1,13 +1,20 @@
 #!/bin/sh
 
-# wait for the buildpack scripts to finish
-timeout=0
-while { ! [ "$(pgrep -f ./agent)" = "" ] && ! [ "$(pgrep -f ./dogstatsd)" = "" ]; } && [ $timeout -lt 120 ]; do
-    sleep 1
-    timeout=$((timeout+1))
-done
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022-Present Datadog, Inc.
 
-# for debugging purposes
-echo $DD_NODE_AGENT_TAGS >> /home/vcap/app/.datadog/dd-node-agent-tags.log
+DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
+DEBUG_FILE="${DATADOG_DIR}/update_agent_config_out.log"
+
+main() {
+    # wait for the buildpack scripts to finish
+    echo "Starting to wait for agent process to start"
+    timeout 120s "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-/bin/bash /home/vcap/app/.datadog/scripts/update_agent_config_restart.sh
+    echo "$DD_NODE_AGENT_TAGS"
+
+    /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
+}
+# for debugging purposes
+main "$@" >> "$DEBUG_FILE" 2>&1

lib/scripts/update_agent_config_restart.sh

@@ -34,23 +34,24 @@ stop_datadog() {
     if [ -f run/agent.pid ]; then
       echo "Stopping agent process, pid: $(cat run/agent.pid)"
       (./agent stop --cfgpath dist/) || true
-      find_pid_kill_and_wait "agent" || true
-      kill_and_wait "run/agent.pid" 5
+      agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
+      find_pid_kill_and_wait "$agent_commad" || true
+      kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5
       rm -f "run/agent.pid"
     fi
 
-    if [ -f run/trace.pid ]; then
+    if [ -f run/trace-agent.pid ]; then
       echo "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
-      trace_agent_command="trace-agent"
-      kill_and_wait "run/trace-agent.pid" 5 1
-      find_pid_kill_and_wait $trace_agent_command "run/trace-agent.pid"
+      trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
+      kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
+      find_pid_kill_and_wait "$trace_agent_command" "${DATADOG_DIR}/run/trace-agent.pid"
     fi
 
     if [ -f run/dogstatsd.pid ]; then
       echo "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
-      dogstatsd_command="dogstatsd"
-      kill_and_wait "run/dogstatsd.pid" 5 1
-      find_pid_kill_and_wait $dogstatsd_command "run/dogstatsd.pid"
+      dogstatsd_command="./dogstatsd start --cfgpath dist/"
+      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
+      find_pid_kill_and_wait "$dogstatsd_command" "${DATADOG_DIR}/run/dogstatsd.pid"
     fi
   popd
 }