From 62b2af874c7599f2bc122969fc1c05f8f39a6b95 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Thu, 8 Dec 2022 20:20:16 +0100
Subject: [PATCH 01/54] WIP

---
 bin/compile                                |  2 +
 bin/supply                                 |  2 +
 lib/redirect-logs.sh                       | 25 +----------
 lib/scripts/check_datadog.sh               | 12 ++---
 lib/scripts/common.sh                      |  9 ++++
 lib/scripts/update_agent_config.sh         | 13 ++++--
 lib/scripts/update_agent_config_restart.sh | 47 +++++++++++++++-----
 lib/scripts/utils.sh                       | 51 ++++++++++++++++------
 8 files changed, 105 insertions(+), 56 deletions(-)
 create mode 100644 lib/scripts/common.sh

diff --git a/bin/compile b/bin/compile
index 5518a38..d459e06 100755
--- a/bin/compile
+++ b/bin/compile
@@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
+cp "${ROOT_DIR}/lib/scripts/common.sh" "${DATADOG_DIR}/scripts/common.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
@@ -49,6 +50,7 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
 fi
 
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
+chmod +x "${DATADOG_DIR}/scripts/common.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
diff --git a/bin/supply b/bin/supply
index 673395c..7daebe6 100755
--- a/bin/supply
+++ b/bin/supply
@@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
+cp "${ROOT_DIR}/lib/scripts/common.sh" "${DATADOG_DIR}/scripts/common.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
@@ -49,6 +50,7 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
 fi
 
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
+chmod +x "${DATADOG_DIR}/scripts/common.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
diff --git a/lib/redirect-logs.sh b/lib/redirect-logs.sh
index 13d408c..90a611e 100644
--- a/lib/redirect-logs.sh
+++ b/lib/redirect-logs.sh
@@ -9,6 +9,8 @@ export STD_LOG_COLLECTION_PORT
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 
+source "${DATADOG_DIR}/scripts/utils.sh"
+
 DD_EU_API_SITE="https://api.datadoghq.eu/api/"
 DD_US_API_SITE="https://api.datadoghq.com/api/"
 DD_API_SITE=$DD_US_API_SITE
@@ -17,29 +19,6 @@ if [ -n "$DD_SITE" ] && [ "$DD_SITE" = "datadoghq.eu" ]; then
   DD_API_SITE=$DD_EU_API_SITE
 fi
 
-# redirect forwards all standard inputs to a TCP socket listening on port STD_LOG_COLLECTION_PORT.
-redirect() {
-  while kill -0 $$; do
-    if [ "$DD_SPARSE_APP_LOGS" = "true" ]; then
-        python "${DATADOG_DIR}/scripts/nc.py" "$STD_LOG_COLLECTION_PORT" || sleep 0.5
-    else
-        nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
-    fi
-    echo "Resetting buildpack log redirection"
-    if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
-      HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
-      -X POST -H "Content-type: application/json" \
-      -d "{
-            \"title\": \"Resetting buildpack log redirection\",
-            \"text\": \"TCP socket on port $STD_LOG_COLLECTION_PORT for log redirection closed. Restarting it.\",
-            \"priority\": \"normal\",
-            \"tags\": $(python ${DATADOG_DIR}/scripts/get_tags.py),
-            \"alert_type\": \"info\"
-      }" "${DD_API_SITE}v1/events?api_key=$DD_API_KEY"
-    fi
-  done
-}
-
 # setup the redirection from stdout/stderr to the logs-agent.
 if [ "$DD_LOGS_ENABLED" = "true" ]; then
   if [ "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 47da33e..e7529b0 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -6,21 +6,23 @@
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 
+source "${DATADOG_DIR}/scripts/common.sh"
+
 check_datadog() {
   while true; do
-    echo "Waiting for agent or dogstatsd process to start"
+    log_message $0 "Waiting for agent or dogstatsd process to start"
     if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
-        echo "Found agent process"
+        log_message $0 "Found agent process"
         if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
-          echo "Found agent token"
+          log_message $0 "Found agent token"
           break
         else 
-          echo "Agent token not found"
+          log_message $0 "Agent token not found"
         fi
     fi
 
     if [ -f "${DATADOG_DIR}/run/dogstatsd.pid" ]; then
-        echo "Found dogstatsd process"
+        log_message $0 "Found dogstatsd process"
         break
     fi
     sleep 1
diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
new file mode 100644
index 0000000..00a8561
--- /dev/null
+++ b/lib/scripts/common.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
+
+log_message() {
+  local component=$1
+  local message=$2
+  echo "$(date +'%d/%m/%Y %H:%M:%S') [${1#/home/vcap/app/}] - ${3:-INFO} - $2"
+}
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index a1bd643..f8eb9e5 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -4,17 +4,24 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2022-Present Datadog, Inc.
 
+
+
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 DEBUG_FILE="${DATADOG_DIR}/update_agent_config_out.log"
 
+. "${DATADOG_DIR}/scripts/common.sh"
+
+
 main() {
     # wait for the buildpack scripts to finish
-    echo "Starting to wait for agent process to start"
+    log_message $0 "Starting to wait for agent process to start"
     timeout 120s "${DATADOG_DIR}/scripts/check_datadog.sh"
 
     echo "$DD_NODE_AGENT_TAGS"
 
-    /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
+    /bin/bash -x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 }
 # for debugging purposes
-main "$@" >> "$DEBUG_FILE" 2>&1
+main "$@" 2>&1 | tee -a "$DEBUG_FILE" 
+
+
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 72bcc6d..107a514 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -12,15 +12,20 @@
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 
+source "${DATADOG_DIR}/scripts/common.sh"
+
+export DD_NODE_AGENT_TAGS_NEW=${DD_NODE_AGENT_TAGS}
+
 # correct way to export / source the .datadog_env file so that every variable is parsed
 python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.datadog_env" "${DATADOG_DIR}/.new_datadog_env"
 source "${DATADOG_DIR}/.new_datadog_env"
 
+export DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS_NEW}
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
 
 # the agent cloud_foundry_container workloadmeta collector reads from this file
 # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
-echo "$DD_TAGS" | awk '{ printf "%s", $0 }' > "${DATADOG_DIR}/node_agent_tags.txt"
+echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
 
 # for debugging purposes
 printenv > "${DATADOG_DIR}/.sourced_datadog_env"
@@ -32,7 +37,7 @@ stop_datadog() {
   pushd "${DATADOG_DIR}"
     # first try to stop the agent so we don't lose data and then force it
     if [ -f run/agent.pid ]; then
-      echo "Stopping agent process, pid: $(cat run/agent.pid)"
+      log_message $0 "Stopping agent process, pid: $(cat run/agent.pid)"
       (./agent stop --cfgpath dist/) || true
       agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
       find_pid_kill_and_wait "$agent_commad" || true
@@ -41,17 +46,19 @@ stop_datadog() {
     fi
 
     if [ -f run/trace-agent.pid ]; then
-      echo "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
+      log_message $0 "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
       trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
       kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
       find_pid_kill_and_wait "$trace_agent_command" "${DATADOG_DIR}/run/trace-agent.pid"
+      rm -f "run/trace-agent.pid"
     fi
 
     if [ -f run/dogstatsd.pid ]; then
-      echo "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
+      log_message $0 "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
       kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
       find_pid_kill_and_wait "$dogstatsd_command" "${DATADOG_DIR}/run/dogstatsd.pid"
+      rm -f "run/dogstatsd.pid"
     fi
   popd
 }
@@ -70,12 +77,12 @@ start_datadog() {
 
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
       if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-        echo "Log endpoint not valid, not starting agent"
+        log_message $0 "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
-        echo "Starting Datadog agent"
+        log_message $0 "Starting Datadog agent"
         python scripts/create_logs_config.py
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
@@ -85,7 +92,7 @@ start_datadog() {
         fi
       fi
     else
-      echo "Starting dogstatsd agent"
+      log_message $0 "Starting dogstatsd agent"
       export DD_LOG_FILE=dogstatsd.log
       if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
@@ -94,7 +101,7 @@ start_datadog() {
       fi
       echo $! > run/dogstatsd.pid
     fi
-    echo "Starting trace agent"
+    log_message $0 "Starting trace agent"
     if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
     else
@@ -104,13 +111,29 @@ start_datadog() {
 }
 
 
-
 main() {
     # After the tags are parsed and added to DD_TAGS, we need to restart the agent for the changes to take effect
-    echo "stop datadog to refresh tags"
+    log_message $0 "stop datadog to refresh tags"
     stop_datadog
-    echo "start datadog to refresh tags"
+
+    # setup the redirection from stdout/stderr to the logs-agent.
+    if [ "$DD_LOGS_ENABLED" = "true" ]; then
+        if [ "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
+            echo "Log endpoint not valid, not starting log redirection"
+        else
+            if [ -z "$LOGS_CONFIG" ]; then
+                echo "can't collect logs, LOGS_CONFIG is not set"
+                else
+                echo "collect all logs for config $LOGS_CONFIG"
+                if [ -n "$STD_LOG_COLLECTION_PORT" ]; then
+                    echo "forward all logs from stdout/stderr to agent port $STD_LOG_COLLECTION_PORT"
+                    exec &> >(tee >(redirect))
+                fi
+            fi
+        fi
+    fi
+    log_message $0 "start datadog to refresh tags"
     start_datadog
 }
 
-main "$@" >> "${DATADOG_DIR}/update_script.log" 2>&1
+main "$@"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index ac869d4..611cada 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -7,18 +7,20 @@
 # These utils are taken from
 # https://github.com/DataDog/datadog-agent-boshrelease/blob/4.11.2/src/helpers/lib.sh
 
-function wait_pid {
+. "${DATADOG_DIR}/scripts/common.sh"
+
+wait_pid() {
   local pidfile=$1
   local pid=$2
   local try_kill=$3
   local timeout=${4:-0}
   local force=${5:-0}
-  local countdown=$(( $timeout * 10 ))
+  local countdown=$(( 100 )) # temporary to workaround a /bin/dash syntax error
   local ps_out="$(ps ax | grep $pid | grep -v grep)"
 
   if [ -e /proc/$pid -o -n "$ps_out" ]; then
     if [ "$try_kill" = "1" ]; then
-      echo "Killing $pidfile: $pid "
+      log_message $0 "Killing $pidfile: $pid "
       kill $pid
     fi
     while [ -e /proc/$pid ]; do
@@ -28,7 +30,7 @@ function wait_pid {
         if [ $countdown -eq 0 ]; then
           if [ "$force" = "1" ]; then
             echo
-            echo "Kill timed out, using kill -9 on $pid ..."
+            log_message $0 "Kill timed out, using kill -9 on $pid ..."
             kill -9 $pid
             sleep 0.5
           fi
@@ -39,22 +41,22 @@ function wait_pid {
       fi
     done
     if [ -e /proc/$pid ]; then
-      echo "Timed Out"
+      log_message $0 "Timed Out"
     else
-      echo "Stopped $pid"
+      log_message $0 "Stopped $pid"
     fi
   else
-    echo "Process $pid is not running"
+    log_message $0 "Process $pid is not running"
   fi
 }
 
-function find_pid {
+find_pid() {
   local find_command=$1
   local pid=$(pgrep -f $find_command)
-  echo $pid
+  log_message $0 $pid
 }
 
-function wait_pidfile {
+wait_pidfile() {
   local pidfile=$1
   local try_kill=$2
   local timeout=${3:-0}
@@ -73,7 +75,7 @@ function wait_pidfile {
   fi
 }
 
-function kill_and_wait {
+kill_and_wait() {
   local pidfile=$1
   local timeout=${2:-25}
   local force=${3:-1}
@@ -88,15 +90,38 @@ function kill_and_wait {
 }
 
 
-function find_pid_kill_and_wait {
+find_pid_kill_and_wait() {
   local find_command=$1
   local pidfile=$2
   local pid=$(find_pid $find_command)
   if [ -z "$pid" ] || [ "$pid" = "" ]; then
-    echo "No such PID $pid exists, skipping the hard kill"
+    log_message $0 "No such PID $pid exists, skipping the hard kill"
   else
     local timeout=${2:-25}
     local force=${3:-1}
     wait_pid $pidfile $pid 1 $timeout $force
   fi
 }
+
+# redirect forwards all standard inputs to a TCP socket listening on port STD_LOG_COLLECTION_PORT.
+redirect() {
+  while kill -0 $$; do
+    if [ "$DD_SPARSE_APP_LOGS" = "true" ]; then
+        python "${DATADOG_DIR}/scripts/nc.py" "$STD_LOG_COLLECTION_PORT" || sleep 0.5
+    else
+        nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
+    fi
+    log_message $0 "Resetting buildpack log redirection"
+    if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
+      HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
+      -X POST -H "Content-type: application/json" \
+      -d "{
+            \"title\": \"Resetting buildpack log redirection\",
+            \"text\": \"TCP socket on port $STD_LOG_COLLECTION_PORT for log redirection closed. Restarting it.\",
+            \"priority\": \"normal\",
+            \"tags\": $(python ${DATADOG_DIR}/scripts/get_tags.py),
+            \"alert_type\": \"info\"
+      }" "${DD_API_SITE}v1/events?api_key=$DD_API_KEY"
+    fi
+  done
+}

From 972978b7b06870ceefc6942cc3d7e2571fdfb0d7 Mon Sep 17 00:00:00 2001
From: Sarah <sarah.witt@datadoghq.com>
Date: Thu, 8 Dec 2022 17:20:02 -0500
Subject: [PATCH 02/54] Clean up variable usage, actually delete pid files

---
 lib/scripts/common.sh                      |   2 +-
 lib/scripts/update_agent_config.sh         |   2 +-
 lib/scripts/update_agent_config_restart.sh |  12 +--
 lib/scripts/utils.sh                       | 104 ++++++++++-----------
 4 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index 00a8561..fc3b8c7 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -5,5 +5,5 @@ DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 log_message() {
   local component=$1
   local message=$2
-  echo "$(date +'%d/%m/%Y %H:%M:%S') [${1#/home/vcap/app/}] - ${3:-INFO} - $2"
+  echo "$(date +'%d/%m/%Y %H:%M:%S') [${1#/home/vcap/app/}] - ${3:-INFO} - ${2}"
 }
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index f8eb9e5..6cc1541 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -19,7 +19,7 @@ main() {
 
     echo "$DD_NODE_AGENT_TAGS"
 
-    /bin/bash -x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
+    /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 }
 # for debugging purposes
 main "$@" 2>&1 | tee -a "$DEBUG_FILE" 
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 107a514..7314645 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -37,27 +37,27 @@ stop_datadog() {
   pushd "${DATADOG_DIR}"
     # first try to stop the agent so we don't lose data and then force it
     if [ -f run/agent.pid ]; then
-      log_message $0 "Stopping agent process, pid: $(cat run/agent.pid)"
+      log_message "${0}" "Stopping agent process, pid: $(cat run/agent.pid)"
       (./agent stop --cfgpath dist/) || true
       agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
       find_pid_kill_and_wait "$agent_commad" || true
-      kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5
+      kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
       rm -f "run/agent.pid"
     fi
 
     if [ -f run/trace-agent.pid ]; then
-      log_message $0 "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
+      log_message "${0}" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
       trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
       kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
-      find_pid_kill_and_wait "$trace_agent_command" "${DATADOG_DIR}/run/trace-agent.pid"
+      find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid"
       rm -f "run/trace-agent.pid"
     fi
 
     if [ -f run/dogstatsd.pid ]; then
-      log_message $0 "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
+      log_message "${0}" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
       kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
-      find_pid_kill_and_wait "$dogstatsd_command" "${DATADOG_DIR}/run/dogstatsd.pid"
+      find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid"
       rm -f "run/dogstatsd.pid"
     fi
   popd
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 611cada..1f34b5b 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -10,96 +10,96 @@
 . "${DATADOG_DIR}/scripts/common.sh"
 
 wait_pid() {
-  local pidfile=$1
-  local pid=$2
-  local try_kill=$3
-  local timeout=${4:-0}
-  local force=${5:-0}
+  local pidfile="${1}"
+  local pid="${2}"
+  local try_kill="${3}"
+  local timeout="${4:-0}"
+  local force="${5:-0}"
   local countdown=$(( 100 )) # temporary to workaround a /bin/dash syntax error
-  local ps_out="$(ps ax | grep $pid | grep -v grep)"
+  local ps_out="$(ps ax | grep ${pid} | grep -v grep)"
 
-  if [ -e /proc/$pid -o -n "$ps_out" ]; then
-    if [ "$try_kill" = "1" ]; then
-      log_message $0 "Killing $pidfile: $pid "
-      kill $pid
+  if [ -e "/proc/${pid}" -o -n "${ps_out}" ]; then
+    if [ "${try_kill}" = "1" ]; then
+      log_message "${0}" "Killing ${pidfile}: ${pid}"
+      kill "${pid}"
     fi
-    while [ -e /proc/$pid ]; do
+    while [ -e "/proc/${pid}" ]; do
       sleep 0.1
-      [ "$countdown" != '0' -a $(( $countdown % 10 )) = '0' ] && echo -n .
-      if [ $timeout -gt 0 ]; then
-        if [ $countdown -eq 0 ]; then
-          if [ "$force" = "1" ]; then
+      [ "${countdown}" != '0' -a $(( "${countdown}" % 10 )) = '0' ] && echo -n .
+      if [ "${timeout}" -gt 0 ]; then
+        if [ "${countdown}" -eq 0 ]; then
+          if [ "${force}" = "1" ]; then
             echo
-            log_message $0 "Kill timed out, using kill -9 on $pid ..."
-            kill -9 $pid
+            log_message "${0}" "Kill timed out, using kill -9 on ${pid} ..."
+            kill -9 "${pid}"
             sleep 0.5
           fi
           break
         else
-          countdown=$(( $countdown - 1 ))
+          countdown=$(( "${countdown}" - 1 ))
         fi
       fi
     done
-    if [ -e /proc/$pid ]; then
-      log_message $0 "Timed Out"
+    if [ -e "/proc/${pid}" ]; then
+      log_message "${0}" "Timed Out"
     else
-      log_message $0 "Stopped $pid"
+      log_message "${0}" "Stopped ${pid}"
     fi
   else
-    log_message $0 "Process $pid is not running"
+    log_message "${0}" "Process ${pid} is not running"
   fi
 }
 
 find_pid() {
-  local find_command=$1
-  local pid=$(pgrep -f $find_command)
-  log_message $0 $pid
+  local find_command="${1}"
+  local pid=$(pgrep -f "${find_command}")
+  echo "${pid:-None}"
 }
 
 wait_pidfile() {
-  local pidfile=$1
-  local try_kill=$2
-  local timeout=${3:-0}
-  local force=${4:-0}
-  local countdown=$(( $timeout * 10 ))
+  local pidfile="${1}"
+  local try_kill="${2}"
+  local timeout="${3:-0}"
+  local force="${4:-0}"
+  local countdown=$(( "${timeout}" * 10 ))
 
-  if [ -f "$pidfile" ]; then
-    pid=$(head -1 "$pidfile")
-    if [ -z "$pid" ]; then
-      die "Unable to get pid from $pidfile"
+  if [ -f "${pidfile}" ]; then
+    pid=$(head -1 "${pidfile}")
+    if [ -z "${pid}" ]; then
+      die "Unable to get pid from ${pidfile}"
     fi
-    wait_pid $pidfile $pid $try_kill $timeout $force
-    rm -f $pidfile
+    wait_pid "${pidfile}" "${pid}" "${try_kill}" "${timeout}" "${force}"
+    rm -f "${pidfile}"
   else
-    printf_log "Pidfile $pidfile doesn't exist"
+    printf_log "Pidfile ${pidfile} doesn't exist"
   fi
 }
 
 kill_and_wait() {
-  local pidfile=$1
-  local timeout=${2:-25}
-  local force=${3:-1}
+  local pidfile="${1}"
+  local timeout="${2:-25}"
+  local force="${3:-1}"
 
   if [ -f "${pidfile}" ]; then
-    wait_pidfile $pidfile 1 $timeout $force
+    wait_pidfile "${pidfile}" 1 "${timeout}" "${force}"
   else
     # TODO assume $1 is something to grep from 'ps ax'
-    pid="$(ps auwwx | grep "'$1'" | awk '{print $2}')"
-    wait_pid $pidfile $pid 1 $timeout $force
+    pid="$(ps auwwx | grep "'${1}'" | awk '{print $2}')"
+    wait_pid "${pidfile}" "${pid}" 1 "${timeout}" "${force}"
   fi
 }
 
 
 find_pid_kill_and_wait() {
-  local find_command=$1
-  local pidfile=$2
-  local pid=$(find_pid $find_command)
-  if [ -z "$pid" ] || [ "$pid" = "" ]; then
-    log_message $0 "No such PID $pid exists, skipping the hard kill"
+  local find_command="${1}"
+  local pidfile="${2}"
+  local pid=$(find_pid "${find_command}")
+  if [ -z "${pid}" ] || [ "${pid}" = "" ] ||  [ "${pid}" = "None" ]; then
+    log_message "${0}" "No such PID ${pid} exists, skipping the hard kill"
   else
-    local timeout=${2:-25}
-    local force=${3:-1}
-    wait_pid $pidfile $pid 1 $timeout $force
+    local timeout="${2:-25}"
+    local force="${3:-1}"
+    wait_pid "${pidfile}" "${pid}" 1 "${timeout}" "${force}"
   fi
 }
 
@@ -111,7 +111,7 @@ redirect() {
     else
         nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
     fi
-    log_message $0 "Resetting buildpack log redirection"
+    log_message "${0}" "Resetting buildpack log redirection"
     if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
       HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
       -X POST -H "Content-type: application/json" \

From 3dce7d0764711b4e23ed2fbbf45e0b8416f01388 Mon Sep 17 00:00:00 2001
From: Sarah <sarah.witt@datadoghq.com>
Date: Thu, 8 Dec 2022 17:27:25 -0500
Subject: [PATCH 03/54] Add header

---
 lib/scripts/common.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index fc3b8c7..1d22db2 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022-Present Datadog, Inc.
+
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 
 log_message() {

From 4250dcd8a9690ea146e23959f3294bc267767c02 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 9 Dec 2022 10:03:56 +0100
Subject: [PATCH 04/54] small nit

---
 lib/scripts/common.sh |  2 +-
 lib/scripts/utils.sh  | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index 1d22db2..f2468c2 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -9,5 +9,5 @@ DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 log_message() {
   local component=$1
   local message=$2
-  echo "$(date +'%d/%m/%Y %H:%M:%S') [${1#/home/vcap/app/}] - ${3:-INFO} - ${2}"
+  echo "$(date +'%d/%m/%Y %H:%M:%S') [${1#/home/vcap/app/}] - ${3:-INFO} - $2"
 }
\ No newline at end of file
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 1f34b5b..55d8b19 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -10,9 +10,9 @@
 . "${DATADOG_DIR}/scripts/common.sh"
 
 wait_pid() {
-  local pidfile="${1}"
-  local pid="${2}"
-  local try_kill="${3}"
+  local pidfile="$1"
+  local pid="$2"
+  local try_kill="$3"
   local timeout="${4:-0}"
   local force="${5:-0}"
   local countdown=$(( 100 )) # temporary to workaround a /bin/dash syntax error
@@ -51,14 +51,14 @@ wait_pid() {
 }
 
 find_pid() {
-  local find_command="${1}"
+  local find_command="$1"
   local pid=$(pgrep -f "${find_command}")
   echo "${pid:-None}"
 }
 
 wait_pidfile() {
-  local pidfile="${1}"
-  local try_kill="${2}"
+  local pidfile="$1"
+  local try_kill="$2"
   local timeout="${3:-0}"
   local force="${4:-0}"
   local countdown=$(( "${timeout}" * 10 ))
@@ -76,7 +76,7 @@ wait_pidfile() {
 }
 
 kill_and_wait() {
-  local pidfile="${1}"
+  local pidfile="$1"
   local timeout="${2:-25}"
   local force="${3:-1}"
 
@@ -84,15 +84,15 @@ kill_and_wait() {
     wait_pidfile "${pidfile}" 1 "${timeout}" "${force}"
   else
     # TODO assume $1 is something to grep from 'ps ax'
-    pid="$(ps auwwx | grep "'${1}'" | awk '{print $2}')"
+    pid="$(ps auwwx | grep "'$1'" | awk '{print $2}')"
     wait_pid "${pidfile}" "${pid}" 1 "${timeout}" "${force}"
   fi
 }
 
 
 find_pid_kill_and_wait() {
-  local find_command="${1}"
-  local pidfile="${2}"
+  local find_command="$1"
+  local pidfile="$2"
   local pid=$(find_pid "${find_command}")
   if [ -z "${pid}" ] || [ "${pid}" = "" ] ||  [ "${pid}" = "None" ]; then
     log_message "${0}" "No such PID ${pid} exists, skipping the hard kill"

From 77862abc82d17f8d43d6a8bcd00617fe8bede527 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 9 Dec 2022 10:38:27 +0100
Subject: [PATCH 05/54] another nit

---
 lib/scripts/update_agent_config_restart.sh |  6 +++---
 lib/scripts/utils.sh                       | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 7314645..927c216 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -37,7 +37,7 @@ stop_datadog() {
   pushd "${DATADOG_DIR}"
     # first try to stop the agent so we don't lose data and then force it
     if [ -f run/agent.pid ]; then
-      log_message "${0}" "Stopping agent process, pid: $(cat run/agent.pid)"
+      log_message "$0" "Stopping agent process, pid: $(cat run/agent.pid)"
       (./agent stop --cfgpath dist/) || true
       agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
       find_pid_kill_and_wait "$agent_commad" || true
@@ -46,7 +46,7 @@ stop_datadog() {
     fi
 
     if [ -f run/trace-agent.pid ]; then
-      log_message "${0}" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
+      log_message "$0" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
       trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
       kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
       find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid"
@@ -54,7 +54,7 @@ stop_datadog() {
     fi
 
     if [ -f run/dogstatsd.pid ]; then
-      log_message "${0}" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
+      log_message "$0" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
       kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
       find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 55d8b19..f56f6e2 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -20,7 +20,7 @@ wait_pid() {
 
   if [ -e "/proc/${pid}" -o -n "${ps_out}" ]; then
     if [ "${try_kill}" = "1" ]; then
-      log_message "${0}" "Killing ${pidfile}: ${pid}"
+      log_message "$0" "Killing ${pidfile}: ${pid}"
       kill "${pid}"
     fi
     while [ -e "/proc/${pid}" ]; do
@@ -30,7 +30,7 @@ wait_pid() {
         if [ "${countdown}" -eq 0 ]; then
           if [ "${force}" = "1" ]; then
             echo
-            log_message "${0}" "Kill timed out, using kill -9 on ${pid} ..."
+            log_message "$0" "Kill timed out, using kill -9 on ${pid} ..."
             kill -9 "${pid}"
             sleep 0.5
           fi
@@ -41,12 +41,12 @@ wait_pid() {
       fi
     done
     if [ -e "/proc/${pid}" ]; then
-      log_message "${0}" "Timed Out"
+      log_message "$0" "Timed Out"
     else
-      log_message "${0}" "Stopped ${pid}"
+      log_message "$0" "Stopped ${pid}"
     fi
   else
-    log_message "${0}" "Process ${pid} is not running"
+    log_message "$0" "Process ${pid} is not running"
   fi
 }
 
@@ -95,7 +95,7 @@ find_pid_kill_and_wait() {
   local pidfile="$2"
   local pid=$(find_pid "${find_command}")
   if [ -z "${pid}" ] || [ "${pid}" = "" ] ||  [ "${pid}" = "None" ]; then
-    log_message "${0}" "No such PID ${pid} exists, skipping the hard kill"
+    log_message "$0" "No such PID ${pid} exists, skipping the hard kill"
   else
     local timeout="${2:-25}"
     local force="${3:-1}"
@@ -111,7 +111,7 @@ redirect() {
     else
         nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
     fi
-    log_message "${0}" "Resetting buildpack log redirection"
+    log_message "$0" "Resetting buildpack log redirection"
     if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
       HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
       -X POST -H "Content-type: application/json" \

From 707d3726594096e73de1fdbfcecc36bdc7308646 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 10 Dec 2022 14:57:36 +0100
Subject: [PATCH 06/54] move env vars processing into compile & supply scripts

---
 bin/compile | 5 ++++-
 bin/supply  | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/bin/compile b/bin/compile
index d459e06..0f2de06 100755
--- a/bin/compile
+++ b/bin/compile
@@ -61,4 +61,7 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
 # Export all environment variables to a file that scripts can use later
 # We need to handle quotes and comments; sed command found in https://gist.github.com/mihow/9c7f559807069a03e302605691f85572
-printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${DATADOG_DIR}/.datadog_env"
+printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$BUILD_DIR/.datadog/.raw_datadog_env"
+
+# sanitize env vars and export a new a env file
+python "$DATADOG_DIR/scripts/parse_env_vars.py" "$DATADOG_DIR/.raw_datadog_env" "$DATADOG_DIR/.datadog_env"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index 7daebe6..1d80033 100755
--- a/bin/supply
+++ b/bin/supply
@@ -61,4 +61,7 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
 # Export all environment variables to a file that scripts can use later
 # We need to handle quotes and comments; sed command found in https://gist.github.com/mihow/9c7f559807069a03e302605691f85572
-printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${DATADOG_DIR}/.datadog_env"
+printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$BUILD_DIR/.datadog/.raw_datadog_env"
+
+# sanitize env vars and export a new a env file
+python "$DATADOG_DIR/scripts/parse_env_vars.py" "$DATADOG_DIR/.raw_datadog_env" "$DATADOG_DIR/.datadog_env"
\ No newline at end of file

From 73b3c72ae351d42115a9340e6247ef1230ffc07b Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 12 Dec 2022 13:52:31 +0100
Subject: [PATCH 07/54] WIP: fix log redirection, added script to update agent
 config tags

---
 bin/compile                                |  2 +
 bin/supply                                 |  5 +-
 lib/run-datadog.sh                         | 77 +++++++++++++++++-----
 lib/scripts/check_datadog.sh               | 10 +--
 lib/scripts/common.sh                      |  7 +-
 lib/scripts/update_agent_config.sh         | 37 +++++++++--
 lib/scripts/update_agent_config_restart.sh | 39 ++++-------
 lib/scripts/update_yaml_config.rb          | 24 +++++++
 8 files changed, 144 insertions(+), 57 deletions(-)
 create mode 100644 lib/scripts/update_yaml_config.rb

diff --git a/bin/compile b/bin/compile
index 0f2de06..50c5a31 100755
--- a/bin/compile
+++ b/bin/compile
@@ -32,6 +32,8 @@ cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
+cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
 cp "${ROOT_DIR}/lib/scripts/common.sh" "${DATADOG_DIR}/scripts/common.sh"
diff --git a/bin/supply b/bin/supply
index 1d80033..14e7f26 100755
--- a/bin/supply
+++ b/bin/supply
@@ -31,6 +31,8 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
+cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
@@ -64,4 +66,5 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$BUILD_DIR/.datadog/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
-python "$DATADOG_DIR/scripts/parse_env_vars.py" "$DATADOG_DIR/.raw_datadog_env" "$DATADOG_DIR/.datadog_env"
\ No newline at end of file
+python "$DATADOG_DIR/scripts/parse_env_vars.py" "$DATADOG_DIR/.raw_datadog_env" "$DATADOG_DIR/.datadog_env"
+
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 79d2977..af44f2c 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -9,9 +9,11 @@ SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 LOCKFILE="${DATADOG_DIR}/lock"
 
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+FIRST_RUN=${FIRST_RUN:-true}
+
+setup_datadog() {
+  pushd "${DATADOG_DIR}"
 
-start_datadog() {
-  pushd ${DATADOG_DIR}
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
     export DD_API_KEY
     export DD_DD_URL
@@ -90,6 +92,7 @@ start_datadog() {
 
     # DSD requires its own config file
     cp dist/datadog.yaml dist/dogstatsd.yaml
+
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
       if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
         echo "Log endpoint not valid, not starting agent"
@@ -97,6 +100,39 @@ start_datadog() {
         export DD_LOG_FILE=${DATADOG_DIR}/agent.log
         export DD_IOT_HOST=false
         sed -i "s~log_file: AGENT_LOG_FILE~log_file: $DD_LOG_FILE~" dist/datadog.yaml
+      fi
+    else
+      export DD_LOG_FILE=${DATADOG_DIR}/dogstatsd.log
+      sed -i "s~log_file: AGENT_LOG_FILE~log_file: $DD_LOG_FILE~" dist/datadog.yaml
+    fi
+  popd
+}
+
+start_datadog() {
+  pushd ${DATADOG_DIR}
+    export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
+    export DD_API_KEY
+    export DD_DD_URL
+    export DD_ENABLE_CHECKS="${DD_ENABLE_CHECKS:-false}"
+    export DOCKER_DD_AGENT=yes
+    export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
+    export LOGS_CONFIG
+
+    if [ "${FIRST_RUN}" = "true" ]; then
+      echo "[DEBUG] First run DATADOG"
+      setup_datadog
+      FIRST_RUN=false
+    else
+      echo "[DEBUG] Not A FIRST run DATADOG"
+      unset DD_TAGS
+    fi
+    
+    if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
+      if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
+        echo "Log endpoint not valid, not starting agent"
+      else
+        export DD_LOG_FILE=${DATADOG_DIR}/agent.log
+        export DD_IOT_HOST=false
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
         else
@@ -105,7 +141,6 @@ start_datadog() {
       fi
     else
       export DD_LOG_FILE=${DATADOG_DIR}/dogstatsd.log
-      sed -i "s~log_file: AGENT_LOG_FILE~log_file: $DD_LOG_FILE~" dist/datadog.yaml
       if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
       else
@@ -122,26 +157,38 @@ start_datadog() {
 }
 
 stop_datadog() {
-  while kill -0 $$; do
+  while kill -0 $$ && [ -f ${DATADOG_DIR}/run/trace-agent.pid ]; do
     sleep 1
   done
-  echo "main process exited, stopping agent"
-  for pidfile in "${DATADOG_DIR}"/run/*; do
-    kill $(cat $pidfile)
-  done
+
+  if ! kill -0 $$; then
+    echo "main process exited, stopping agent"
+    for pidfile in "${DATADOG_DIR}"/run/*; do
+      kill $(cat $pidfile)
+    done
+    exit
+  fi
+}
+
+
+monit_datadog() {
+  while true; do
+      exec 9> "$LOCKFILE" || exit 1
+      if flock -x -n 9; then
+        source "${DATADOG_DIR}/.sourced_datadog_env"
+        echo "starting datadog"
+        start_datadog
+        stop_datadog  &
+        exec 9>&-
+      fi
+    done
 }
 
 main() {
   if [ -z "$DD_API_KEY" ]; then
     echo "Datadog API Key not set, not starting Datadog"
   else
-    exec 9> "$LOCKFILE" || exit 1
-    if flock -x -n 9; then
-      echo "starting datadog"
-      start_datadog
-      stop_datadog &
-      exec 9>&-
-    fi
+    monit_datadog &
   fi
 }
 
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index e7529b0..9222e22 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -10,19 +10,19 @@ source "${DATADOG_DIR}/scripts/common.sh"
 
 check_datadog() {
   while true; do
-    log_message $0 "Waiting for agent or dogstatsd process to start"
+    log_message $0 $$ "Waiting for agent or dogstatsd process to start"
     if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
-        log_message $0 "Found agent process"
+        log_message $0 $$ "Found agent process"
         if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
-          log_message $0 "Found agent token"
+          log_message $0 $$ "Found agent token"
           break
         else 
-          log_message $0 "Agent token not found"
+          log_message $0 $$ "Agent token not found"
         fi
     fi
 
     if [ -f "${DATADOG_DIR}/run/dogstatsd.pid" ]; then
-        log_message $0 "Found dogstatsd process"
+        log_message $0 $$ "Found dogstatsd process"
         break
     fi
     sleep 1
diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index f2468c2..f74dc7c 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -4,10 +4,11 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2022-Present Datadog, Inc.
 
-DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
+export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
+export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 
 log_message() {
   local component=$1
-  local message=$2
-  echo "$(date +'%d/%m/%Y %H:%M:%S') [${1#/home/vcap/app/}] - ${3:-INFO} - $2"
+  local message=$3
+  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${1#/home/vcap/app/}][PID:$2] - ${4:-INFO} - $3"
 }
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 6cc1541..5011834 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -7,21 +7,46 @@
 
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
-DEBUG_FILE="${DATADOG_DIR}/update_agent_config_out.log"
+LOCKFILE="${DATADOG_DIR}/update.lock"
 
 . "${DATADOG_DIR}/scripts/common.sh"
 
+release_lock() {
+    log_message $0 $$ "Releasing LOCKFILE"
+    rmdir "$LOCKFILE"
+}
 
 main() {
+    log_message $0 $$ "Starting Update Script"
+
+    # try to create the LOCKFILE
+    while ! mkdir "$LOCKFILE" 2>/dev/null; do
+        log_message $0 $$ "Script is already running"
+    done
+
+    # ensures the lock is released on exit
+    trap release_lock INT TERM EXIT
+
+    log_message $0 $$ "Starting to wait for agent process to start"
+
     # wait for the buildpack scripts to finish
-    log_message $0 "Starting to wait for agent process to start"
     timeout 120s "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-    echo "$DD_NODE_AGENT_TAGS"
+    export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
+
+    # the agent cloud_foundry_container workloadmeta collector reads from this file
+    # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
+    echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
+
+
+    echo "running ruby script"
+    /usr/bin/env ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb
+
+    log_message $0 $$ "$DD_NODE_AGENT_TAGS"
 
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
+
+    log_message $0 $$ "Finished Update Script"
 }
 # for debugging purposes
-main "$@" 2>&1 | tee -a "$DEBUG_FILE" 
-
-
+main "$@" 2>&1 | tee /dev/fd/1 -a "$DEBUG_FILE"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 927c216..abd25a3 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -12,13 +12,13 @@
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 
+# import log_message and DATADOG_DIR
 source "${DATADOG_DIR}/scripts/common.sh"
 
 export DD_NODE_AGENT_TAGS_NEW=${DD_NODE_AGENT_TAGS}
 
-# correct way to export / source the .datadog_env file so that every variable is parsed
-python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.datadog_env" "${DATADOG_DIR}/.new_datadog_env"
-source "${DATADOG_DIR}/.new_datadog_env"
+# correct way to export / source the .datadog_env file so that every variable is parsed and sanitized
+source "${DATADOG_DIR}/.datadog_env"
 
 export DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS_NEW}
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
@@ -30,6 +30,9 @@ echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.t
 # for debugging purposes
 printenv > "${DATADOG_DIR}/.sourced_datadog_env"
 
+echo "running ruby script"
+ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+
 # import helper functions
 source "${DATADOG_DIR}/scripts/utils.sh"
 
@@ -77,12 +80,12 @@ start_datadog() {
 
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
       if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-        log_message $0 "Log endpoint not valid, not starting agent"
+        log_message $0 $$ "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
-        log_message $0 "Starting Datadog agent"
+        log_message $0 $$ "Starting Datadog agent"
         python scripts/create_logs_config.py
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
@@ -92,7 +95,7 @@ start_datadog() {
         fi
       fi
     else
-      log_message $0 "Starting dogstatsd agent"
+      log_message $0 $$ "Starting dogstatsd agent"
       export DD_LOG_FILE=dogstatsd.log
       if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
@@ -101,7 +104,7 @@ start_datadog() {
       fi
       echo $! > run/dogstatsd.pid
     fi
-    log_message $0 "Starting trace agent"
+    log_message $0 $$ "Starting trace agent"
     if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
     else
@@ -113,27 +116,9 @@ start_datadog() {
 
 main() {
     # After the tags are parsed and added to DD_TAGS, we need to restart the agent for the changes to take effect
-    log_message $0 "stop datadog to refresh tags"
+    log_message $0 $$ "stop datadog to refresh tags"
     stop_datadog
-
-    # setup the redirection from stdout/stderr to the logs-agent.
-    if [ "$DD_LOGS_ENABLED" = "true" ]; then
-        if [ "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-            echo "Log endpoint not valid, not starting log redirection"
-        else
-            if [ -z "$LOGS_CONFIG" ]; then
-                echo "can't collect logs, LOGS_CONFIG is not set"
-                else
-                echo "collect all logs for config $LOGS_CONFIG"
-                if [ -n "$STD_LOG_COLLECTION_PORT" ]; then
-                    echo "forward all logs from stdout/stderr to agent port $STD_LOG_COLLECTION_PORT"
-                    exec &> >(tee >(redirect))
-                fi
-            fi
-        fi
-    fi
-    log_message $0 "start datadog to refresh tags"
-    start_datadog
+    log_message $0 $$ "start datadog to refresh tags"
 }
 
 main "$@"
diff --git a/lib/scripts/update_yaml_config.rb b/lib/scripts/update_yaml_config.rb
new file mode 100644
index 0000000..2079732
--- /dev/null
+++ b/lib/scripts/update_yaml_config.rb
@@ -0,0 +1,24 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022-Present Datadog, Inc.
+
+#!/usr/bin/env ruby
+
+require 'yaml'
+
+# env vars
+DATADOG_DIR = ENV.fetch('DATADOG_DIR', '/home/vcap/app/.datadog')
+DD_TAGS = ENV.fetch('DD_TAGS', '')
+DD_NODE_AGENT_TAGS = ENV.fetch('DD_NODE_AGENT_TAGS', '')
+
+
+datadog_config_filepath = File.join(DATADOG_DIR, 'dist/datadog.yaml')
+
+
+file = File.open(datadog_config_filepath, 'r')
+yaml_data = YAML.load(file)
+
+tags = DD_TAGS.split(',') + DD_NODE_AGENT_TAGS.split(',')
+yaml_data['tags'] = tags.uniq
+
+File.write(datadog_config_filepath, yaml_data.to_yaml)

From 57d3da718f9985aa0f25b55fd370fecdd7c55fd1 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Mon, 12 Dec 2022 22:17:59 -0500
Subject: [PATCH 08/54] Fix parsing of commands, use correct arguments, update
 start_agent

---
 lib/run-datadog.sh                         | 22 +++++++++++++++------
 lib/scripts/check_datadog.sh               | 10 +++++-----
 lib/scripts/common.sh                      |  6 +++---
 lib/scripts/update_agent_config.sh         | 18 +++++++++--------
 lib/scripts/update_agent_config_restart.sh | 23 +++++++++++-----------
 5 files changed, 46 insertions(+), 33 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index af44f2c..07af82c 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -126,13 +126,17 @@ start_datadog() {
       echo "[DEBUG] Not A FIRST run DATADOG"
       unset DD_TAGS
     fi
-    
+
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
       if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-        echo "Log endpoint not valid, not starting agent"
+        log_message "$0" "$$" "Log endpoint not valid, not starting agent"
       else
-        export DD_LOG_FILE=${DATADOG_DIR}/agent.log
+        export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
+
+        log_message "$0" "$$" "Starting Datadog agent"
+        python scripts/create_logs_config.py
+
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
         else
@@ -140,7 +144,8 @@ start_datadog() {
         fi
       fi
     else
-      export DD_LOG_FILE=${DATADOG_DIR}/dogstatsd.log
+      log_message "$0" "$$" "Starting dogstatsd agent"
+      export DD_LOG_FILE=dogstatsd.log
       if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
       else
@@ -148,6 +153,7 @@ start_datadog() {
       fi
       echo $! > run/dogstatsd.pid
     fi
+    log_message "$0" "$$" "Starting trace agent"
     if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
     else
@@ -175,8 +181,12 @@ monit_datadog() {
   while true; do
       exec 9> "$LOCKFILE" || exit 1
       if flock -x -n 9; then
-        source "${DATADOG_DIR}/.sourced_datadog_env"
-        echo "starting datadog"
+        if [ -f "${DATADOG_DIR}/.sourced_datadog_env" ]; then
+          source"${DATADOG_DIR}/.sourced_datadog_env"
+        else 
+          source "${DATADOG_DIR}/.datadog_env"
+        fi
+        echo "starting datadog hey"
         start_datadog
         stop_datadog  &
         exec 9>&-
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 9222e22..d42155e 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -10,19 +10,19 @@ source "${DATADOG_DIR}/scripts/common.sh"
 
 check_datadog() {
   while true; do
-    log_message $0 $$ "Waiting for agent or dogstatsd process to start"
+    log_message "$0" "$$" "Waiting for agent or dogstatsd process to start"
     if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
-        log_message $0 $$ "Found agent process"
+        log_message "$0" "$$" "Found agent process"
         if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
-          log_message $0 $$ "Found agent token"
+          log_message "$0" "$$" "Found agent token"
           break
         else 
-          log_message $0 $$ "Agent token not found"
+          log_message "$0" "$$" "Agent token not found"
         fi
     fi
 
     if [ -f "${DATADOG_DIR}/run/dogstatsd.pid" ]; then
-        log_message $0 $$ "Found dogstatsd process"
+        log_message "$0" "$$" "Found dogstatsd process"
         break
     fi
     sleep 1
diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index f74dc7c..8434056 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -8,7 +8,7 @@ export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 
 log_message() {
-  local component=$1
-  local message=$3
-  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${1#/home/vcap/app/}][PID:$2] - ${4:-INFO} - $3"
+  local component="${1}"
+  local message="${3}"
+  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${1#/home/vcap/app/}][PID:$2] - ${4:-INFO} - ${3}"
 }
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 5011834..b046f93 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -4,7 +4,7 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2022-Present Datadog, Inc.
 
-
+set -x 
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 LOCKFILE="${DATADOG_DIR}/update.lock"
@@ -12,22 +12,22 @@ LOCKFILE="${DATADOG_DIR}/update.lock"
 . "${DATADOG_DIR}/scripts/common.sh"
 
 release_lock() {
-    log_message $0 $$ "Releasing LOCKFILE"
+    log_message "$0" "$$" "Releasing LOCKFILE"
     rmdir "$LOCKFILE"
 }
 
 main() {
-    log_message $0 $$ "Starting Update Script"
+    log_message "$0" "$$" "Starting Update Script"
 
     # try to create the LOCKFILE
     while ! mkdir "$LOCKFILE" 2>/dev/null; do
-        log_message $0 $$ "Script is already running"
+        log_message "$0" "$$" "Script is already running"
     done
 
     # ensures the lock is released on exit
     trap release_lock INT TERM EXIT
 
-    log_message $0 $$ "Starting to wait for agent process to start"
+    log_message "$0" "$$" "Starting to wait for agent process to start"
 
     # wait for the buildpack scripts to finish
     timeout 120s "${DATADOG_DIR}/scripts/check_datadog.sh"
@@ -40,13 +40,15 @@ main() {
 
 
     echo "running ruby script"
-    /usr/bin/env ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb
+    #/usr/bin/env ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb
+
+    log_message "$0" "$$" "$DD_NODE_AGENT_TAGS"
+    log_message "$0" "$$" "$DD_TAGS"
 
-    log_message $0 $$ "$DD_NODE_AGENT_TAGS"
 
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 
-    log_message $0 $$ "Finished Update Script"
+    log_message "$0" "$$" "Finished Update Script"
 }
 # for debugging purposes
 main "$@" 2>&1 | tee /dev/fd/1 -a "$DEBUG_FILE"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index abd25a3..e2cb806 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -8,7 +8,7 @@
 # It sets the DD_NODE_AGENT_TAGS environment variable with these new tags
 # see: https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/cloudfoundry/containertagger/container_tagger.go#L131
 
-
+set -x 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 
@@ -43,7 +43,7 @@ stop_datadog() {
       log_message "$0" "Stopping agent process, pid: $(cat run/agent.pid)"
       (./agent stop --cfgpath dist/) || true
       agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-      find_pid_kill_and_wait "$agent_commad" || true
+      find_pid_kill_and_wait "$agent_commad" "${DATADOG_DIR}run/agent.pid" 5 1 || true
       kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
       rm -f "run/agent.pid"
     fi
@@ -52,15 +52,15 @@ stop_datadog() {
       log_message "$0" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
       trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
       kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
-      find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid"
+      find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid" 5 1
       rm -f "run/trace-agent.pid"
     fi
 
     if [ -f run/dogstatsd.pid ]; then
       log_message "$0" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
-      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
-      find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid"
+      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 0 0
+      find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 
       rm -f "run/dogstatsd.pid"
     fi
   popd
@@ -80,12 +80,12 @@ start_datadog() {
 
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
       if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-        log_message $0 $$ "Log endpoint not valid, not starting agent"
+        log_message "$0" "$$" "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
-        log_message $0 $$ "Starting Datadog agent"
+        log_message "$0" "$$" "Starting Datadog agent"
         python scripts/create_logs_config.py
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
@@ -95,7 +95,7 @@ start_datadog() {
         fi
       fi
     else
-      log_message $0 $$ "Starting dogstatsd agent"
+      log_message "$0" "$$" "Starting dogstatsd agent"
       export DD_LOG_FILE=dogstatsd.log
       if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
@@ -104,7 +104,7 @@ start_datadog() {
       fi
       echo $! > run/dogstatsd.pid
     fi
-    log_message $0 $$ "Starting trace agent"
+    log_message "$0" "$$" "Starting trace agent"
     if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
     else
@@ -116,9 +116,10 @@ start_datadog() {
 
 main() {
     # After the tags are parsed and added to DD_TAGS, we need to restart the agent for the changes to take effect
-    log_message $0 $$ "stop datadog to refresh tags"
+    log_message "$0" "$$" "stop datadog to refresh tags"
     stop_datadog
-    log_message $0 $$ "start datadog to refresh tags"
+    log_message "$0" "$$" "start datadog to refresh tags"
+    start_datadog
 }
 
 main "$@"

From 9bdc4d09471ea64c55556a7dcffaa2a8337072d3 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Mon, 12 Dec 2022 22:36:28 -0500
Subject: [PATCH 09/54] Don't add None tag

---
 lib/scripts/get_tags.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/scripts/get_tags.py b/lib/scripts/get_tags.py
index ca39c0e..81f0d75 100644
--- a/lib/scripts/get_tags.py
+++ b/lib/scripts/get_tags.py
@@ -17,7 +17,10 @@
 
 cf_instance_ip = os.environ.get("CF_INSTANCE_IP")
 
-tags = ["cf_instance_ip:{}".format(cf_instance_ip)]
+tags = []
+if cf_instance_ip is not None:
+    tags.append("cf_instance_ip:{}".format(cf_instance_ip))
+
 tags.append("container_id:{}".format(os.environ.get("CF_INSTANCE_GUID")))
 
 if len(sys.argv) > 1 and sys.argv[1] == 'node-agent-tags':

From b330a25c14fb3cab4429a0cc76723784083702c2 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Mon, 12 Dec 2022 23:07:38 -0500
Subject: [PATCH 10/54] Fix kill_and_wait

---
 lib/scripts/update_agent_config.sh         | 2 --
 lib/scripts/update_agent_config_restart.sh | 3 +--
 lib/scripts/utils.sh                       | 4 ++--
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index b046f93..8acc99e 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -4,8 +4,6 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2022-Present Datadog, Inc.
 
-set -x 
-
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 LOCKFILE="${DATADOG_DIR}/update.lock"
 
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index e2cb806..c67c96e 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -8,7 +8,6 @@
 # It sets the DD_NODE_AGENT_TAGS environment variable with these new tags
 # see: https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/cloudfoundry/containertagger/container_tagger.go#L131
 
-set -x 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 
@@ -59,7 +58,7 @@ stop_datadog() {
     if [ -f run/dogstatsd.pid ]; then
       log_message "$0" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
-      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 0 0
+      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
       find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 
       rm -f "run/dogstatsd.pid"
     fi
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index f56f6e2..dc143d4 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -97,8 +97,8 @@ find_pid_kill_and_wait() {
   if [ -z "${pid}" ] || [ "${pid}" = "" ] ||  [ "${pid}" = "None" ]; then
     log_message "$0" "No such PID ${pid} exists, skipping the hard kill"
   else
-    local timeout="${2:-25}"
-    local force="${3:-1}"
+    local timeout="${3:-25}"
+    local force="${4:-1}"
     wait_pid "${pidfile}" "${pid}" 1 "${timeout}" "${force}"
   fi
 }

From 02057aea6134b6abcf99c9d6c12e0b1bc0ca981b Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Mon, 12 Dec 2022 23:42:02 -0500
Subject: [PATCH 11/54] Fix command

---
 lib/scripts/update_agent_config_restart.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index c67c96e..fd09211 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -42,7 +42,7 @@ stop_datadog() {
       log_message "$0" "Stopping agent process, pid: $(cat run/agent.pid)"
       (./agent stop --cfgpath dist/) || true
       agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-      find_pid_kill_and_wait "$agent_commad" "${DATADOG_DIR}run/agent.pid" 5 1 || true
+      find_pid_kill_and_wait "$agent_commad" "${DATADOG_DIR}/run/agent.pid" 5 1 || true
       kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
       rm -f "run/agent.pid"
     fi

From 54510de2431905adc0432f4bf97562139c8af806 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Tue, 13 Dec 2022 18:17:28 +0100
Subject: [PATCH 12/54] working tags update

---
 lib/run-datadog.sh                 | 16 +++++++++-------
 lib/scripts/update_agent_config.sh | 13 ++++++++-----
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 07af82c..31fa8a5 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -129,12 +129,12 @@ start_datadog() {
 
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
       if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-        log_message "$0" "$$" "Log endpoint not valid, not starting agent"
+        echo "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
-        log_message "$0" "$$" "Starting Datadog agent"
+        echo "Starting Datadog agent"
         python scripts/create_logs_config.py
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
@@ -144,7 +144,7 @@ start_datadog() {
         fi
       fi
     else
-      log_message "$0" "$$" "Starting dogstatsd agent"
+      echo "Starting dogstatsd agent"
       export DD_LOG_FILE=dogstatsd.log
       if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
@@ -153,7 +153,7 @@ start_datadog() {
       fi
       echo $! > run/dogstatsd.pid
     fi
-    log_message "$0" "$$" "Starting trace agent"
+    echo "Starting trace agent"
     if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
     else
@@ -172,6 +172,7 @@ stop_datadog() {
     for pidfile in "${DATADOG_DIR}"/run/*; do
       kill $(cat $pidfile)
     done
+    exec 9>&-
     exit
   fi
 }
@@ -182,14 +183,15 @@ monit_datadog() {
       exec 9> "$LOCKFILE" || exit 1
       if flock -x -n 9; then
         if [ -f "${DATADOG_DIR}/.sourced_datadog_env" ]; then
-          source"${DATADOG_DIR}/.sourced_datadog_env"
+          source "${DATADOG_DIR}/.sourced_datadog_env"
         else 
           source "${DATADOG_DIR}/.datadog_env"
         fi
         echo "starting datadog hey"
         start_datadog
-        stop_datadog  &
-        exec 9>&-
+        stop_datadog
+      else
+        exit
       fi
     done
 }
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 8acc99e..0d798aa 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -32,16 +32,19 @@ main() {
 
     export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
 
+    log_message "$0" "$$" "DD_TAGS=${DD_TAGS}"
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
-    echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
-
+    log_message "$0" "$$" "Writing DD_TAGS to node_agent_tags.txt"
+    echo "${DD_TAGS}" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
+    log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
+    
 
     echo "running ruby script"
-    #/usr/bin/env ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb
+    # /usr/bin/env ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb
 
-    log_message "$0" "$$" "$DD_NODE_AGENT_TAGS"
-    log_message "$0" "$$" "$DD_TAGS"
+    log_message "$0" "$$" "DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
+  
 
 
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"

From 90de8f4bd23e1e4d43c7c0d677dc7e9007510de7 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 14 Dec 2022 11:26:46 +0100
Subject: [PATCH 13/54] cleanup log_message function

---
 lib/scripts/common.sh                      | 8 +++++---
 lib/scripts/update_agent_config.sh         | 2 --
 lib/scripts/update_agent_config_restart.sh | 4 ++--
 lib/scripts/utils.sh                       | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index 8434056..083a486 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -8,7 +8,9 @@ export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 
 log_message() {
-  local component="${1}"
-  local message="${3}"
-  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${1#/home/vcap/app/}][PID:$2] - ${4:-INFO} - ${3}"
+  local component="${1#/home/vcap/app/}"
+  local pid="$2"
+  local message="$3"
+  local log_level="${4:-INFO}"
+  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${component}][PID:${pid}] - ${log_level} - ${message}"
 }
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 0d798aa..18c5045 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -45,8 +45,6 @@ main() {
 
     log_message "$0" "$$" "DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
   
-
-
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 
     log_message "$0" "$$" "Finished Update Script"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index fd09211..2891b19 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -117,8 +117,8 @@ main() {
     # After the tags are parsed and added to DD_TAGS, we need to restart the agent for the changes to take effect
     log_message "$0" "$$" "stop datadog to refresh tags"
     stop_datadog
-    log_message "$0" "$$" "start datadog to refresh tags"
-    start_datadog
+    # log_message "$0" "$$" "start datadog to refresh tags"
+    # start_datadog
 }
 
 main "$@"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index dc143d4..3f1d7ed 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -111,7 +111,7 @@ redirect() {
     else
         nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
     fi
-    log_message "$0" "Resetting buildpack log redirection"
+    log_message "$0" "$$" "Resetting buildpack log redirection"
     if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
       HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
       -X POST -H "Content-type: application/json" \

From 6312cc3f61c2382cfb4761408bbd511e07ee05ef Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 14 Dec 2022 16:48:10 +0100
Subject: [PATCH 14/54] working logs redireciton, workings logs, missing
 metadata tags

---
 bin/compile                                |  1 +
 bin/supply                                 |  1 +
 lib/run-datadog.sh                         | 85 +++++++++++++-------
 lib/scripts/check_datadog.sh               |  1 +
 lib/scripts/common.sh                      |  5 ++
 lib/scripts/create_logs_config.rb          | 43 ++++++++++
 lib/scripts/update_agent_config.sh         | 18 ++++-
 lib/scripts/update_agent_config_restart.sh | 92 +---------------------
 8 files changed, 125 insertions(+), 121 deletions(-)
 create mode 100644 lib/scripts/create_logs_config.rb

diff --git a/bin/compile b/bin/compile
index 50c5a31..194aca6 100755
--- a/bin/compile
+++ b/bin/compile
@@ -31,6 +31,7 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
+cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 
diff --git a/bin/supply b/bin/supply
index 14e7f26..ad636cd 100755
--- a/bin/supply
+++ b/bin/supply
@@ -31,6 +31,7 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
+cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 31fa8a5..b14f005 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -8,6 +8,11 @@ DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 LOCKFILE="${DATADOG_DIR}/lock"
 
+AGENT_CMD="./agent run --cfgpath dist/ --pidfile run/agent.pid"
+DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
+TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
+
+
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 FIRST_RUN=${FIRST_RUN:-true}
 
@@ -35,7 +40,7 @@ setup_datadog() {
     # add logs configs
     if [ -n "$LOGS_CONFIG" ]; then
       mkdir -p $LOGS_CONFIG_DIR
-      python scripts/create_logs_config.py
+      ruby scripts/create_logs_config.rb
     fi
 
     # The yaml file requires the tags to be an array,
@@ -124,7 +129,7 @@ start_datadog() {
       FIRST_RUN=false
     else
       echo "[DEBUG] Not A FIRST run DATADOG"
-      unset DD_TAGS
+      source "${DATADOG_DIR}/.sourced_datadog_env"
     fi
 
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
@@ -135,7 +140,7 @@ start_datadog() {
         export DD_IOT_HOST=false
 
         echo "Starting Datadog agent"
-        python scripts/create_logs_config.py
+        ruby scripts/create_logs_config.rb
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
@@ -162,46 +167,66 @@ start_datadog() {
   popd
 }
 
+
 stop_datadog() {
-  while kill -0 $$ && [ -f ${DATADOG_DIR}/run/trace-agent.pid ]; do
-    sleep 1
-  done
+  pushd "${DATADOG_DIR}"
+    if pgrep "${AGENT_CMD}"; then
+      log_message "$0" "Stopping agent process, pid: $(cat run/agent.pid)"
+      # first try to stop the agent so we don't lose data and then force it
+      (./agent stop --cfgpath dist/) || true
+      agent_command="./agent run --cfgpath dist/ --pidfile run/agent.pid"
+      find_pid_kill_and_wait "$agent_command" "${DATADOG_DIR}/run/agent.pid" 5 1 || true
+      kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
+      rm -f "run/agent.pid"
+    fi
 
-  if ! kill -0 $$; then
-    echo "main process exited, stopping agent"
-    for pidfile in "${DATADOG_DIR}"/run/*; do
-      kill $(cat $pidfile)
-    done
-    exec 9>&-
-    exit
-  fi
+    if pgrep "${DOGSTATSD_CMD}"; then
+      log_message "$0" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
+      dogstatsd_command="./dogstatsd start --cfgpath dist/"
+      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
+      find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 
+      rm -f "run/dogstatsd.pid"
+    fi
+
+    if pgrep "${TRACE_AGENT_CMD}"; then
+      log_message "$0" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
+      trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
+      kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
+      find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid" 5 1
+      rm -f "run/trace-agent.pid"
+    fi
+  popd
 }
 
 
 monit_datadog() {
   while true; do
-      exec 9> "$LOCKFILE" || exit 1
-      if flock -x -n 9; then
-        if [ -f "${DATADOG_DIR}/.sourced_datadog_env" ]; then
-          source "${DATADOG_DIR}/.sourced_datadog_env"
-        else 
-          source "${DATADOG_DIR}/.datadog_env"
-        fi
-        echo "starting datadog hey"
-        start_datadog
-        stop_datadog
-      else
-        exit
-      fi
-    done
+    if ! kill -0 $$; then
+      echo "main process exited, stopping agent"
+      for pidfile in "${DATADOG_DIR}"/run/*; do
+        kill $(cat $pidfile)
+      done
+      exit
+    elif [ -f "${DATADOG_DIR}"/tags_updated ]; then
+      echo "RESTARTING DATADOG"
+      start_datadog
+      rm -f "${DATADOG_DIR}"/tags_updated
+    fi
+    sleep 1
+  done
 }
 
 main() {
   if [ -z "$DD_API_KEY" ]; then
     echo "Datadog API Key not set, not starting Datadog"
   else
-    monit_datadog &
+    exec 9> "$LOCKFILE" || exit 1
+    if flock -x -n 9; then
+      echo "starting datadog"
+      start_datadog
+      monit_datadog &
+      exec 9>&-
+    fi
   fi
 }
-
 main "$@"
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index d42155e..0ff80d3 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -27,6 +27,7 @@ check_datadog() {
     fi
     sleep 1
   done
+  sleep 5
 }
 
 main() {
diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index 083a486..9bb8e4b 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -7,6 +7,11 @@
 export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 
+export AGENT_CMD="./agent run --cfgpath dist/ --pidfile run/agent.pid"
+export DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
+export TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
+
+
 log_message() {
   local component="${1#/home/vcap/app/}"
   local pid="$2"
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
new file mode 100644
index 0000000..288196f
--- /dev/null
+++ b/lib/scripts/create_logs_config.rb
@@ -0,0 +1,43 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2017-Present Datadog, Inc.
+
+require 'json'
+
+logs_config_dir = ENV['LOGS_CONFIG_DIR']
+logs_config = ENV['LOGS_CONFIG']
+dd_tags = ENV['DD_TAGS']
+
+config = {}
+
+if logs_config_dir.nil?
+  puts "ERROR: `LOGS_CONFIG_DIR` must be set in order to collect logs. For more info, see: https://github.com/DataDog/datadog-cloudfoundry-buildpack#log-collection"
+  exit(1)
+end
+
+if !logs_config.nil?
+  config["logs"] = JSON.parse(logs_config)
+  if !dd_tags.nil?
+    config["logs"][0]["tags"] = dd_tags
+  else
+    puts "Could not find DD_TAGS env var"
+  end
+else
+  puts "ERROR: `LOGS_CONFIG` must be set in order to collect logs. For more info, see: https://github.com/DataDog/datadog-cloudfoundry-buildpack#log-collection"
+  exit(1)
+end
+
+config = config.to_json
+
+path = "#{logs_config_dir}/logs.yaml"
+
+begin
+  Dir.mkdir(logs_config_dir) unless File.exists?(logs_config_dir)
+  File.open(path, 'w') do |f|
+    puts "writing #{config} to #{path}"
+    f.write(config)
+    f.write("\n")
+  end
+rescue Exception => e
+  puts "Could not write to log file #{e.backtrace}"
+end
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 18c5045..2facd37 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -20,6 +20,7 @@ main() {
     # try to create the LOCKFILE
     while ! mkdir "$LOCKFILE" 2>/dev/null; do
         log_message "$0" "$$" "Script is already running"
+        sleep 1
     done
 
     # ensures the lock is released on exit
@@ -28,7 +29,20 @@ main() {
     log_message "$0" "$$" "Starting to wait for agent process to start"
 
     # wait for the buildpack scripts to finish
-    timeout 120s "${DATADOG_DIR}/scripts/check_datadog.sh"
+    timeout 300s "${DATADOG_DIR}/scripts/check_datadog.sh"
+
+    # TODO: check for dogstatsd as well
+    if ! [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
+        log_message "$0" "$$" "Could not find agent, aborting update script!"
+        exit 1
+    fi
+
+    if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
+        if ! [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
+          log_message "$0" "$$" "Could not find agent token, aborting update script!"
+          exit 2
+        fi
+    fi
 
     export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
 
@@ -48,6 +62,8 @@ main() {
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 
     log_message "$0" "$$" "Finished Update Script"
+
+    touch $DATADOG_DIR/tags_updated
 }
 # for debugging purposes
 main "$@" 2>&1 | tee /dev/fd/1 -a "$DEBUG_FILE"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 2891b19..90740e2 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -30,95 +30,7 @@ echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.t
 printenv > "${DATADOG_DIR}/.sourced_datadog_env"
 
 echo "running ruby script"
-ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+# ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 
 # import helper functions
-source "${DATADOG_DIR}/scripts/utils.sh"
-
-stop_datadog() {
-  pushd "${DATADOG_DIR}"
-    # first try to stop the agent so we don't lose data and then force it
-    if [ -f run/agent.pid ]; then
-      log_message "$0" "Stopping agent process, pid: $(cat run/agent.pid)"
-      (./agent stop --cfgpath dist/) || true
-      agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-      find_pid_kill_and_wait "$agent_commad" "${DATADOG_DIR}/run/agent.pid" 5 1 || true
-      kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
-      rm -f "run/agent.pid"
-    fi
-
-    if [ -f run/trace-agent.pid ]; then
-      log_message "$0" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
-      trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
-      kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
-      find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid" 5 1
-      rm -f "run/trace-agent.pid"
-    fi
-
-    if [ -f run/dogstatsd.pid ]; then
-      log_message "$0" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
-      dogstatsd_command="./dogstatsd start --cfgpath dist/"
-      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
-      find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 
-      rm -f "run/dogstatsd.pid"
-    fi
-  popd
-}
-
-start_datadog() {
-  pushd "${DATADOG_DIR}"
-
-    export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
-    export DD_API_KEY
-    export DD_DD_URL
-    export DD_ENABLE_CHECKS="${DD_ENABLE_CHECKS:-false}"
-    export DOCKER_DD_AGENT=yes
-    export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
-    export LOGS_CONFIG
-    export DD_API_KEY
-
-    if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
-      if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
-        log_message "$0" "$$" "Log endpoint not valid, not starting agent"
-      else
-        export DD_LOG_FILE=agent.log
-        export DD_IOT_HOST=false
-
-        log_message "$0" "$$" "Starting Datadog agent"
-        python scripts/create_logs_config.py
-
-        if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
-          ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
-        else
-          ./agent run --cfgpath dist/ --pidfile run/agent.pid &
-        fi
-      fi
-    else
-      log_message "$0" "$$" "Starting dogstatsd agent"
-      export DD_LOG_FILE=dogstatsd.log
-      if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
-        ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
-      else
-        ./dogstatsd start --cfgpath dist/ &
-      fi
-      echo $! > run/dogstatsd.pid
-    fi
-    log_message "$0" "$$" "Starting trace agent"
-    if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
-      ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
-    else
-      ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid &
-    fi
-  popd
-}
-
-
-main() {
-    # After the tags are parsed and added to DD_TAGS, we need to restart the agent for the changes to take effect
-    log_message "$0" "$$" "stop datadog to refresh tags"
-    stop_datadog
-    # log_message "$0" "$$" "start datadog to refresh tags"
-    # start_datadog
-}
-
-main "$@"
+source "${DATADOG_DIR}/scripts/utils.sh"
\ No newline at end of file

From 9725991c2d160185e4120906f6e6871e4728d86d Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Thu, 15 Dec 2022 01:41:09 +0100
Subject: [PATCH 15/54] WIP: use pidfiles, improve startup/shutdown management

---
 lib/run-datadog.sh                         | 38 ++++++++++++----------
 lib/scripts/check_datadog.sh               |  4 +--
 lib/scripts/common.sh                      |  5 ++-
 lib/scripts/create_logs_config.rb          | 31 +++++++++++++++++-
 lib/scripts/update_agent_config.sh         |  7 ++--
 lib/scripts/update_agent_config_restart.sh | 11 +++++--
 6 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index b14f005..ab7fa82 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -8,14 +8,12 @@ DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 LOCKFILE="${DATADOG_DIR}/lock"
 
-AGENT_CMD="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
-TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
-
-
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 FIRST_RUN=${FIRST_RUN:-true}
 
+source $DATADOG_DIR/scripts/common.sh
+source $DATADOG_DIR/scripts/utils.sh
+
 setup_datadog() {
   pushd "${DATADOG_DIR}"
 
@@ -40,7 +38,7 @@ setup_datadog() {
     # add logs configs
     if [ -n "$LOGS_CONFIG" ]; then
       mkdir -p $LOGS_CONFIG_DIR
-      ruby scripts/create_logs_config.rb
+      ruby scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.1.log"
     fi
 
     # The yaml file requires the tags to be an array,
@@ -124,12 +122,16 @@ start_datadog() {
     export LOGS_CONFIG
 
     if [ "${FIRST_RUN}" = "true" ]; then
-      echo "[DEBUG] First run DATADOG"
+      echo "First run datadog"
       setup_datadog
       FIRST_RUN=false
     else
-      echo "[DEBUG] Not A FIRST run DATADOG"
-      source "${DATADOG_DIR}/.sourced_datadog_env"
+      echo "Not first run datadog"
+      if [ -f  "${DATADOG_DIR}/.sourced_datadog_env" ]; then
+        source "${DATADOG_DIR}/.sourced_datadog_env"
+      elif [ -f  "${DATADOG_DIR}/.datadog_env" ]; then
+        source "${DATADOG_DIR}/.datadog_env"
+      fi
     fi
 
     if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
@@ -140,7 +142,7 @@ start_datadog() {
         export DD_IOT_HOST=false
 
         echo "Starting Datadog agent"
-        ruby scripts/create_logs_config.rb
+      # ruby scripts/create_logs_config.rb
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
@@ -170,8 +172,8 @@ start_datadog() {
 
 stop_datadog() {
   pushd "${DATADOG_DIR}"
-    if pgrep "${AGENT_CMD}"; then
-      log_message "$0" "Stopping agent process, pid: $(cat run/agent.pid)"
+    if kill -0 $(cat ${DATADOG_DIR}/run/agent.pid) > /dev/null; then
+      echo "Stopping agent process, pid: $(cat run/agent.pid)"
       # first try to stop the agent so we don't lose data and then force it
       (./agent stop --cfgpath dist/) || true
       agent_command="./agent run --cfgpath dist/ --pidfile run/agent.pid"
@@ -180,16 +182,16 @@ stop_datadog() {
       rm -f "run/agent.pid"
     fi
 
-    if pgrep "${DOGSTATSD_CMD}"; then
-      log_message "$0" "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
+    if kill -0 $(cat "${DATADOG_DIR}/run/dogstatsd.pid") > /dev/null; then
+      echo "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
       kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
       find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 
       rm -f "run/dogstatsd.pid"
     fi
 
-    if pgrep "${TRACE_AGENT_CMD}"; then
-      log_message "$0" "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
+    if kill -0 $(cat "${DATADOG_DIR}/run/trace-agent.pid"); then
+      echo "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
       trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
       kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
       find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid" 5 1
@@ -208,7 +210,9 @@ monit_datadog() {
       done
       exit
     elif [ -f "${DATADOG_DIR}"/tags_updated ]; then
-      echo "RESTARTING DATADOG"
+      echo "STOPPING DATADOG"
+      stop_datadog
+      echo "STARTING DATADOG"
       start_datadog
       rm -f "${DATADOG_DIR}"/tags_updated
     fi
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 0ff80d3..40ecbbf 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -11,7 +11,7 @@ source "${DATADOG_DIR}/scripts/common.sh"
 check_datadog() {
   while true; do
     log_message "$0" "$$" "Waiting for agent or dogstatsd process to start"
-    if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
+    if kill -0 $(cat "${DATADOG_DIR}/run/agent.pid"); then
         log_message "$0" "$$" "Found agent process"
         if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
           log_message "$0" "$$" "Found agent token"
@@ -21,7 +21,7 @@ check_datadog() {
         fi
     fi
 
-    if [ -f "${DATADOG_DIR}/run/dogstatsd.pid" ]; then
+    if kill -0 $(cat "${DATADOG_DIR}/run/dogstatsd.pid"); then
         log_message "$0" "$$" "Found dogstatsd process"
         break
     fi
diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
index 9bb8e4b..010c7bc 100644
--- a/lib/scripts/common.sh
+++ b/lib/scripts/common.sh
@@ -7,9 +7,8 @@
 export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 
-export AGENT_CMD="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-export DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
-export TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
+export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
+export LOGS_CONFIG
 
 
 log_message() {
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index 288196f..5b06f6c 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -4,9 +4,21 @@
 
 require 'json'
 
+dd_env_file = "/home/vcap/app/.datadog/.sourced_env_datadog"
+node_agent_tags = "/home/vcap/app/.datadog/node_agent_tags.txt"
+
+if File.file?(dd_env_file)
+  File.readlines().each do |line|
+    values = line.split("=")
+    ENV[values[0]] = values[1]
+  end
+  puts "ruby: sourced datadog env file"
+end
+
 logs_config_dir = ENV['LOGS_CONFIG_DIR']
 logs_config = ENV['LOGS_CONFIG']
 dd_tags = ENV['DD_TAGS']
+dd_node_agent_tags = ENV['DD_NODE_AGENTS_TAGS'] || (File.file?(node_agent_tags) ? File.read(node_agent_tags) : "")
 
 config = {}
 
@@ -17,11 +29,28 @@
 
 if !logs_config.nil?
   config["logs"] = JSON.parse(logs_config)
+
+  tags_list = []
+
   if !dd_tags.nil?
-    config["logs"][0]["tags"] = dd_tags
+    tags_list += dd_tags.split(',')
+    puts "DD_TAGS found in ruby script=#{dd_tags}"
   else
     puts "Could not find DD_TAGS env var"
   end
+
+  if !dd_node_agent_tags.nil?
+    tags_list += dd_node_agent_tags.split(',')
+    puts "DD_NODE_AGENTS_TAGS found in ruby script=#{dd_node_agent_tags}"
+  else
+    puts "Could not find DD_NODE_AGENTS_TAGS env var"
+  end
+
+  if !tags_list.empty?
+    tags_list = tags_list.uniq
+    config["logs"][0]["tags"] = tags_list.join(",")
+  end
+  
 else
   puts "ERROR: `LOGS_CONFIG` must be set in order to collect logs. For more info, see: https://github.com/DataDog/datadog-cloudfoundry-buildpack#log-collection"
   exit(1)
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 2facd37..d85ea05 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -45,6 +45,10 @@ main() {
     fi
 
     export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
+    export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
+    export LOGS_CONFIG
+    echo "running ruby script"
+    ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
 
     log_message "$0" "$$" "DD_TAGS=${DD_TAGS}"
     # the agent cloud_foundry_container workloadmeta collector reads from this file
@@ -54,9 +58,6 @@ main() {
     log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
     
 
-    echo "running ruby script"
-    # /usr/bin/env ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb
-
     log_message "$0" "$$" "DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
   
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 90740e2..3ca3699 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -21,6 +21,13 @@ source "${DATADOG_DIR}/.datadog_env"
 
 export DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS_NEW}
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
+export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
+export LOGS_CONFIG
+
+echo "running ruby script"
+ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb" 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
+
+echo "LOGS_CONFIG: $(cat ${DATADOG_DIR}/dist/conf.d/logs.d/logs.yaml)"
 
 # the agent cloud_foundry_container workloadmeta collector reads from this file
 # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
@@ -29,8 +36,8 @@ echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.t
 # for debugging purposes
 printenv > "${DATADOG_DIR}/.sourced_datadog_env"
 
-echo "running ruby script"
-# ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+
+
 
 # import helper functions
 source "${DATADOG_DIR}/scripts/utils.sh"
\ No newline at end of file

From 847f774bbfd51e410be93e1e4dfc028916606b9a Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Thu, 15 Dec 2022 02:23:53 +0100
Subject: [PATCH 16/54] working tags update, not sure if tags injection is
 stable

---
 lib/run-datadog.sh                         | 2 +-
 lib/scripts/update_agent_config.sh         | 3 ++-
 lib/scripts/update_agent_config_restart.sh | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index ab7fa82..3a35b1e 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -142,7 +142,7 @@ start_datadog() {
         export DD_IOT_HOST=false
 
         echo "Starting Datadog agent"
-      # ruby scripts/create_logs_config.rb
+        ruby scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.4.log"
 
         if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index d85ea05..c64e67e 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -48,7 +48,8 @@ main() {
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
     echo "running ruby script"
-    ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
+    # ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
+    ruby ${DATADOG_DIR}/scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
 
     log_message "$0" "$$" "DD_TAGS=${DD_TAGS}"
     # the agent cloud_foundry_container workloadmeta collector reads from this file
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index 3ca3699..a01b848 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -25,7 +25,8 @@ export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
 export LOGS_CONFIG
 
 echo "running ruby script"
-ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb" 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
+ruby $DATADOG_DIR/scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
+# ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb" 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
 
 echo "LOGS_CONFIG: $(cat ${DATADOG_DIR}/dist/conf.d/logs.d/logs.yaml)"
 

From 0b0a298ed988b6cedff34f7b45c73456fc11760e Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Thu, 15 Dec 2022 12:53:17 +0100
Subject: [PATCH 17/54] everything working

---
 lib/scripts/check_datadog.sh               |  2 +-
 lib/scripts/update_agent_config.sh         | 10 ++++++----
 lib/scripts/update_agent_config_restart.sh | 11 +----------
 3 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 40ecbbf..8ba5f61 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -27,7 +27,7 @@ check_datadog() {
     fi
     sleep 1
   done
-  sleep 5
+  sleep 5 # TODO: use agent status?
 }
 
 main() {
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index c64e67e..11f24e4 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -44,11 +44,13 @@ main() {
         fi
     fi
 
-    export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
+    . $DATADOG_DIR/.datadog_env
+
+    export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)   
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
+
     echo "running ruby script"
-    # ruby ${DATADOG_DIR}/scripts/update_yaml_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
     ruby ${DATADOG_DIR}/scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
 
     log_message "$0" "$$" "DD_TAGS=${DD_TAGS}"
@@ -56,9 +58,9 @@ main() {
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
     log_message "$0" "$$" "Writing DD_TAGS to node_agent_tags.txt"
     echo "${DD_TAGS}" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
+    
     log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
     
-
     log_message "$0" "$$" "DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
   
     /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
@@ -68,4 +70,4 @@ main() {
     touch $DATADOG_DIR/tags_updated
 }
 # for debugging purposes
-main "$@" 2>&1 | tee /dev/fd/1 -a "$DEBUG_FILE"
+main "$@" 2>&1 | tee -a "$DEBUG_FILE"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
index a01b848..33289a8 100644
--- a/lib/scripts/update_agent_config_restart.sh
+++ b/lib/scripts/update_agent_config_restart.sh
@@ -26,19 +26,10 @@ export LOGS_CONFIG
 
 echo "running ruby script"
 ruby $DATADOG_DIR/scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
-# ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb" 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
-
-echo "LOGS_CONFIG: $(cat ${DATADOG_DIR}/dist/conf.d/logs.d/logs.yaml)"
 
 # the agent cloud_foundry_container workloadmeta collector reads from this file
 # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
 echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
 
 # for debugging purposes
-printenv > "${DATADOG_DIR}/.sourced_datadog_env"
-
-
-
-
-# import helper functions
-source "${DATADOG_DIR}/scripts/utils.sh"
\ No newline at end of file
+printenv > "${DATADOG_DIR}/.sourced_datadog_env"
\ No newline at end of file

From f29711ebc43abd9d62bc2656b1fb714128ebb58a Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 16 Dec 2022 17:59:42 +0100
Subject: [PATCH 18/54] cleanup

---
 lib/run-datadog.sh                         | 33 +++++++----
 lib/scripts/check_datadog.sh               |  5 +-
 lib/scripts/create_logs_config.rb          |  4 +-
 lib/scripts/update_agent_config.sh         | 69 ++++++++++++----------
 lib/scripts/update_agent_config_restart.sh | 35 -----------
 lib/scripts/utils.sh                       |  2 +-
 6 files changed, 65 insertions(+), 83 deletions(-)
 delete mode 100644 lib/scripts/update_agent_config_restart.sh

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 3a35b1e..ad8f9fb 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -112,7 +112,7 @@ setup_datadog() {
 }
 
 start_datadog() {
-  pushd ${DATADOG_DIR}
+  pushd "${DATADOG_DIR}"
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
     export DD_API_KEY
     export DD_DD_URL
@@ -122,14 +122,15 @@ start_datadog() {
     export LOGS_CONFIG
 
     if [ "${FIRST_RUN}" = "true" ]; then
-      echo "First run datadog"
+      echo "setting up datadog"
       setup_datadog
       FIRST_RUN=false
     else
-      echo "Not first run datadog"
       if [ -f  "${DATADOG_DIR}/.sourced_datadog_env" ]; then
+        echo "sourcing .sourced_datadog_env file"
         source "${DATADOG_DIR}/.sourced_datadog_env"
       elif [ -f  "${DATADOG_DIR}/.datadog_env" ]; then
+        echo "sourcing .datadog_env file"
         source "${DATADOG_DIR}/.datadog_env"
       fi
     fi
@@ -169,10 +170,21 @@ start_datadog() {
   popd
 }
 
+# check_if_running() {
+#   local pidfile="$1"
+#   local command="${2:-none}"
+  
+#   if [ -f "${pidfile}" ] && 
+#     return kill -0 "$(cat ${pidfile})" > /dev/null; then
+#   else 
+
+#   fi
+# }
 
 stop_datadog() {
   pushd "${DATADOG_DIR}"
-    if kill -0 $(cat ${DATADOG_DIR}/run/agent.pid) > /dev/null; then
+    # TODO: use a fallback approach in case the pid file is missing
+    if kill -0 "$(cat ${DATADOG_DIR}/run/agent.pid)" >/dev/null; then
       echo "Stopping agent process, pid: $(cat run/agent.pid)"
       # first try to stop the agent so we don't lose data and then force it
       (./agent stop --cfgpath dist/) || true
@@ -182,7 +194,7 @@ stop_datadog() {
       rm -f "run/agent.pid"
     fi
 
-    if kill -0 $(cat "${DATADOG_DIR}/run/dogstatsd.pid") > /dev/null; then
+    if kill -0 "$(cat "${DATADOG_DIR}/run/dogstatsd.pid")" > /dev/null; then
       echo "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
       dogstatsd_command="./dogstatsd start --cfgpath dist/"
       kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
@@ -190,7 +202,7 @@ stop_datadog() {
       rm -f "run/dogstatsd.pid"
     fi
 
-    if kill -0 $(cat "${DATADOG_DIR}/run/trace-agent.pid"); then
+    if kill -0 "$(cat "${DATADOG_DIR}/run/trace-agent.pid")" >/dev/null; then
       echo "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
       trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
       kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
@@ -206,15 +218,16 @@ monit_datadog() {
     if ! kill -0 $$; then
       echo "main process exited, stopping agent"
       for pidfile in "${DATADOG_DIR}"/run/*; do
-        kill $(cat $pidfile)
+        kill "$(cat "${pidfile}")"
       done
       exit
     elif [ -f "${DATADOG_DIR}"/tags_updated ]; then
-      echo "STOPPING DATADOG"
+      echo "tags_updated found, stopping datadog agents"
       stop_datadog
-      echo "STARTING DATADOG"
+      echo "tags_updated found, starting datadog agents"
       start_datadog
-      rm -f "${DATADOG_DIR}"/tags_updated
+      echo "deleting tags_updated"
+      rm -f "${DATADOG_DIR}"/tags_updated # TODO: check for race conditions
     fi
     sleep 1
   done
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 8ba5f61..903704d 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -11,7 +11,7 @@ source "${DATADOG_DIR}/scripts/common.sh"
 check_datadog() {
   while true; do
     log_message "$0" "$$" "Waiting for agent or dogstatsd process to start"
-    if kill -0 $(cat "${DATADOG_DIR}/run/agent.pid"); then
+    if kill -0 "$(cat "${DATADOG_DIR}/run/agent.pid")" > /dev/null; then
         log_message "$0" "$$" "Found agent process"
         if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
           log_message "$0" "$$" "Found agent token"
@@ -21,13 +21,12 @@ check_datadog() {
         fi
     fi
 
-    if kill -0 $(cat "${DATADOG_DIR}/run/dogstatsd.pid"); then
+    if kill -0 "$(cat "${DATADOG_DIR}/run/dogstatsd.pid")" > /dev/null; then
         log_message "$0" "$$" "Found dogstatsd process"
         break
     fi
     sleep 1
   done
-  sleep 5 # TODO: use agent status?
 }
 
 main() {
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index 5b06f6c..f6a045f 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -34,14 +34,14 @@
 
   if !dd_tags.nil?
     tags_list += dd_tags.split(',')
-    puts "DD_TAGS found in ruby script=#{dd_tags}"
+    # puts "DD_TAGS found in ruby script=#{dd_tags}"
   else
     puts "Could not find DD_TAGS env var"
   end
 
   if !dd_node_agent_tags.nil?
     tags_list += dd_node_agent_tags.split(',')
-    puts "DD_NODE_AGENTS_TAGS found in ruby script=#{dd_node_agent_tags}"
+    # puts "DD_NODE_AGENTS_TAGS found in ruby script=#{dd_node_agent_tags}"
   else
     puts "Could not find DD_NODE_AGENTS_TAGS env var"
   end
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 11f24e4..525743b 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -5,69 +5,74 @@
 # Copyright 2022-Present Datadog, Inc.
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
-LOCKFILE="${DATADOG_DIR}/update.lock"
+LOCK="${DATADOG_DIR}/update.lock"
 
 . "${DATADOG_DIR}/scripts/common.sh"
 
 release_lock() {
-    log_message "$0" "$$" "Releasing LOCKFILE"
-    rmdir "$LOCKFILE"
+    log_message "$0" "$$" "releasing LOCK ${LOCK}"
+    rmdir "${LOCK}"
 }
 
 main() {
-    log_message "$0" "$$" "Starting Update Script"
+    log_message "$0" "$$" "starting update_agent_config script"
+    log_message "$0" "$$" "(BEFORE)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
 
-    # try to create the LOCKFILE
-    while ! mkdir "$LOCKFILE" 2>/dev/null; do
-        log_message "$0" "$$" "Script is already running"
+    # try to create the LOCK
+    while ! mkdir "${LOCK}" 2>/dev/null; do
+        log_message "$0" "$$" "cannot acquire lock, script is already running"
         sleep 1
     done
 
+    log_message "$0" "$$" "acquired LOCK ${LOCK}"
+   
     # ensures the lock is released on exit
     trap release_lock INT TERM EXIT
 
-    log_message "$0" "$$" "Starting to wait for agent process to start"
-
     # wait for the buildpack scripts to finish
-    timeout 300s "${DATADOG_DIR}/scripts/check_datadog.sh"
-
-    # TODO: check for dogstatsd as well
-    if ! [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
-        log_message "$0" "$$" "Could not find agent, aborting update script!"
-        exit 1
-    fi
-
-    if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
-        if ! [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
-          log_message "$0" "$$" "Could not find agent token, aborting update script!"
-          exit 2
-        fi
+    log_message "$0" "$$" "starting check_datadog script"
+    timeout 300s "${DATADOG_DIR}/scripts/check_datadog.sh" 
+    exit_code=$?
+    log_message "$0" "$$" "finished check_datadog script"
+
+    # verify that check_datadog didn't end because of the timeout
+    if [ ${exit_code} -ne  0 ]; then
+        log_message "$0" "$$" "could not find agent, aborting update script!"
+        exit ${exit_code}
     fi
 
-    . $DATADOG_DIR/.datadog_env
+    # source relevant DD tags
+    . "${DATADOG_DIR}/.datadog_env"
 
-    export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)   
+    # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
+    export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)"
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
 
-    echo "running ruby script"
-    ruby ${DATADOG_DIR}/scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
+    # update logs configs with the new tags
+    log_message "$0" "$$" "running create_logs_config ruby script"
+    ruby "${DATADOG_DIR}/scripts/create_logs_config.rb" 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
 
-    log_message "$0" "$$" "DD_TAGS=${DD_TAGS}"
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
     log_message "$0" "$$" "Writing DD_TAGS to node_agent_tags.txt"
     echo "${DD_TAGS}" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
     
+    # log DD_TAGS and DD_NODE_AGENT_TAGS values
     log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
+    log_message "$0" "$$" "(AFTER)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
+
     
-    log_message "$0" "$$" "DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
-  
-    /bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
+    # finishing up
+    log_message "$0" "$$" "exporting .sourced_datadog_env file"
+    printenv > "${DATADOG_DIR}/.sourced_datadog_env"
 
-    log_message "$0" "$$" "Finished Update Script"
+    # mark to the monit_datadog function in run-datadog.sh that the script is finished
+    log_message "$0" "$$" "creating tags_updated file"
+    touch "${DATADOG_DIR}/tags_updated"
 
-    touch $DATADOG_DIR/tags_updated
+    log_message "$0" "$$" "finished update_agent_config script"
 }
+
 # for debugging purposes
 main "$@" 2>&1 | tee -a "$DEBUG_FILE"
diff --git a/lib/scripts/update_agent_config_restart.sh b/lib/scripts/update_agent_config_restart.sh
deleted file mode 100644
index 33289a8..0000000
--- a/lib/scripts/update_agent_config_restart.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
-# This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2022-Present Datadog, Inc.
-
-# This script is called by the node agent to expose CAPI metadata and DCA tags to the container agents
-# It sets the DD_NODE_AGENT_TAGS environment variable with these new tags
-# see: https://github.com/DataDog/datadog-agent/blob/7.40.x/pkg/cloudfoundry/containertagger/container_tagger.go#L131
-
-DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
-SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
-
-# import log_message and DATADOG_DIR
-source "${DATADOG_DIR}/scripts/common.sh"
-
-export DD_NODE_AGENT_TAGS_NEW=${DD_NODE_AGENT_TAGS}
-
-# correct way to export / source the .datadog_env file so that every variable is parsed and sanitized
-source "${DATADOG_DIR}/.datadog_env"
-
-export DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS_NEW}
-export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)
-export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
-export LOGS_CONFIG
-
-echo "running ruby script"
-ruby $DATADOG_DIR/scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.2.log"
-
-# the agent cloud_foundry_container workloadmeta collector reads from this file
-# See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
-echo "$DD_TAGS" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
-
-# for debugging purposes
-printenv > "${DATADOG_DIR}/.sourced_datadog_env"
\ No newline at end of file
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 3f1d7ed..b377723 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -105,7 +105,7 @@ find_pid_kill_and_wait() {
 
 # redirect forwards all standard inputs to a TCP socket listening on port STD_LOG_COLLECTION_PORT.
 redirect() {
-  while kill -0 $$; do
+  while kill -0 $$ 2>/dev/null; do
     if [ "$DD_SPARSE_APP_LOGS" = "true" ]; then
         python "${DATADOG_DIR}/scripts/nc.py" "$STD_LOG_COLLECTION_PORT" || sleep 0.5
     else

From 6a9aafd917e7b4543918bb8d41ffc82914f7569a Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 16 Dec 2022 22:29:28 +0100
Subject: [PATCH 19/54] merge common.sh into utils.sh + cleanup

---
 lib/run-datadog.sh                 | 18 ++++--------------
 lib/scripts/check_datadog.sh       |  2 +-
 lib/scripts/update_agent_config.sh |  7 +++++--
 lib/scripts/utils.sh               | 24 +++++++++++++++++++++++-
 lib/test-endpoint.sh               |  2 +-
 5 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index ad8f9fb..8236fb8 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -7,12 +7,13 @@
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 LOCKFILE="${DATADOG_DIR}/lock"
+FIRST_RUN="${FIRST_RUN:-true}"
+
+source $DATADOG_DIR/scripts/utils.sh
 
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-FIRST_RUN=${FIRST_RUN:-true}
 
-source $DATADOG_DIR/scripts/common.sh
-source $DATADOG_DIR/scripts/utils.sh
+
 
 setup_datadog() {
   pushd "${DATADOG_DIR}"
@@ -170,17 +171,6 @@ start_datadog() {
   popd
 }
 
-# check_if_running() {
-#   local pidfile="$1"
-#   local command="${2:-none}"
-  
-#   if [ -f "${pidfile}" ] && 
-#     return kill -0 "$(cat ${pidfile})" > /dev/null; then
-#   else 
-
-#   fi
-# }
-
 stop_datadog() {
   pushd "${DATADOG_DIR}"
     # TODO: use a fallback approach in case the pid file is missing
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 903704d..7fb2847 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -6,7 +6,7 @@
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 
-source "${DATADOG_DIR}/scripts/common.sh"
+source "${DATADOG_DIR}/scripts/utils.sh"
 
 check_datadog() {
   while true; do
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 525743b..5173f21 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -1,13 +1,16 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2022-Present Datadog, Inc.
 
+
+
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 LOCK="${DATADOG_DIR}/update.lock"
 
-. "${DATADOG_DIR}/scripts/common.sh"
+# import utils function such as log_message
+. "${DATADOG_DIR}/scripts/utils.sh"
 
 release_lock() {
     log_message "$0" "$$" "releasing LOCK ${LOCK}"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index b377723..9ff25fa 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -7,7 +7,29 @@
 # These utils are taken from
 # https://github.com/DataDog/datadog-agent-boshrelease/blob/4.11.2/src/helpers/lib.sh
 
-. "${DATADOG_DIR}/scripts/common.sh"
+export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
+export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
+export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
+export LOGS_CONFIG
+
+log_message() {
+  local component="${1#/home/vcap/app/}"
+  local pid="$2"
+  local message="$3"
+  local log_level="${4:-INFO}"
+  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${component}][PID:${pid}] - ${log_level} - ${message}"
+}
+
+check_if_running() {
+  local pidfile="$1"
+  local command="${2:-none}"
+  
+  if [ -f "${pidfile}" ]; then
+    kill -0 "$(cat "${pidfile}")" > /dev/null
+  else
+    pgrep -f "${command}"
+  fi
+}
 
 wait_pid() {
   local pidfile="$1"
diff --git a/lib/test-endpoint.sh b/lib/test-endpoint.sh
index 2ccaa14..d561a6e 100644
--- a/lib/test-endpoint.sh
+++ b/lib/test-endpoint.sh
@@ -64,7 +64,7 @@ if [ -z "$DD_LOGS_CONFIG_LOGS_DD_URL" ]; then
   fi
 fi
 
-if [ "$DD_LOGS_ENABLED" = "true" -a -n $DD_LOGS_CONFIG_LOGS_DD_URL -a "$DD_SKIP_LOGS_TEST" != "true" ]; then
+if [ "${DD_LOGS_ENABLED}" = "true" -a -n $DD_LOGS_CONFIG_LOGS_DD_URL -a "$DD_SKIP_LOGS_TEST" != "true" ]; then
   echo "Validating log endpoint $DD_LOGS_CONFIG_LOGS_DD_URL"
   LOGS_ENDPOINT=`echo $DD_LOGS_CONFIG_LOGS_DD_URL | cut -d ":" -f1`
   LOGS_PORT=`echo $DD_LOGS_CONFIG_LOGS_DD_URL | cut -d ":" -f2`

From 43358805e041ba52844698369f0a81eaf363d9fa Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 16 Dec 2022 22:39:38 +0100
Subject: [PATCH 20/54] use curly braces for all env vars

---
 docker/build.sh                    |  8 ++--
 docker/run.sh                      |  4 +-
 lib/redirect-logs.sh               | 18 ++++----
 lib/run-datadog.sh                 | 72 +++++++++++++++---------------
 lib/scripts/update_agent_config.sh |  4 +-
 lib/scripts/utils.sh               | 14 +++---
 lib/test-endpoint.sh               | 58 ++++++++++++------------
 7 files changed, 89 insertions(+), 89 deletions(-)

diff --git a/docker/build.sh b/docker/build.sh
index 34bbb03..30c5254 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -2,18 +2,18 @@
 
 DOCKER_DIR=$(cd "$(dirname $0)/." && pwd)
 
-ROOT_DIR="$DOCKER_DIR/.."
+ROOT_DIR="${DOCKER_DIR}/.."
 
 # First, build the buildpack.
 # The build script is the easiest way to get the buildpack assets
 # This script allows the assets to be grabbed without building the buildpack itself
 
-if [ -z "$DO_NOT_BUILD" ]; then
-  pushd $ROOT_DIR
+if [ -z "${DO_NOT_BUILD}" ]; then
+  pushd ${ROOT_DIR}
     NO_ZIP=true REFRESH_ASSETS=true ./build
   popd
 fi
 
 # Build the container
 
-docker build $ROOT_DIR -t docker_app_with_buildpack -f $DOCKER_DIR/Dockerfile
+docker build ${ROOT_DIR} -t docker_app_with_buildpack -f ${DOCKER_DIR}/Dockerfile
diff --git a/docker/run.sh b/docker/run.sh
index 6ca5e6f..3a1763a 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -2,10 +2,10 @@
 
 # This is a simple script to run the application.
 # Used only for testing your docker image locally.
-# You will need to have the $DD_API_KEY available in order to run it
+# You will need to have the ${DD_API_KEY} available in order to run it
 
 docker run --rm \
-  -e DD_API_KEY="$DD_API_KEY" \
+  -e DD_API_KEY="${DD_API_KEY}" \
   -e PORT=5050 \
   -p 5050:5050 \
   docker_app_with_buildpack
diff --git a/lib/redirect-logs.sh b/lib/redirect-logs.sh
index 90a611e..9b297b8 100644
--- a/lib/redirect-logs.sh
+++ b/lib/redirect-logs.sh
@@ -13,23 +13,23 @@ source "${DATADOG_DIR}/scripts/utils.sh"
 
 DD_EU_API_SITE="https://api.datadoghq.eu/api/"
 DD_US_API_SITE="https://api.datadoghq.com/api/"
-DD_API_SITE=$DD_US_API_SITE
+DD_API_SITE=${DD_US_API_SITE}
 
-if [ -n "$DD_SITE" ] && [ "$DD_SITE" = "datadoghq.eu" ]; then
-  DD_API_SITE=$DD_EU_API_SITE
+if [ -n "${DD_SITE}" ] && [ "${DD_SITE}" = "datadoghq.eu" ]; then
+  DD_API_SITE=${DD_EU_API_SITE}
 fi
 
 # setup the redirection from stdout/stderr to the logs-agent.
-if [ "$DD_LOGS_ENABLED" = "true" ]; then
-  if [ "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
+if [ "${DD_LOGS_ENABLED}" = "true" ]; then
+  if [ "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
     echo "Log endpoint not valid, not starting log redirection"
   else
-    if [ -z "$LOGS_CONFIG" ]; then
+    if [ -z "${LOGS_CONFIG}" ]; then
       echo "can't collect logs, LOGS_CONFIG is not set"
     else
-      echo "collect all logs for config $LOGS_CONFIG"
-      if [ -n "$STD_LOG_COLLECTION_PORT" ]; then
-        echo "forward all logs from stdout/stderr to agent port $STD_LOG_COLLECTION_PORT"
+      echo "collect all logs for config ${LOGS_CONFIG}"
+      if [ -n "${STD_LOG_COLLECTION_PORT}" ]; then
+        echo "forward all logs from stdout/stderr to agent port ${STD_LOG_COLLECTION_PORT}"
         exec &> >(tee >(redirect))
       fi
     fi
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 8236fb8..6959d77 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -9,7 +9,7 @@ SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 
-source $DATADOG_DIR/scripts/utils.sh
+source ${DATADOG_DIR}/scripts/utils.sh
 
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 
@@ -27,19 +27,19 @@ setup_datadog() {
     export LOGS_CONFIG
 
     # create and configure set /conf.d if integrations are enabled
-    if [ "$DD_ENABLE_CHECKS" = "true" ] || [ -n "$LOGS_CONFIG" ] ; then
+    if [ "${DD_ENABLE_CHECKS}" = "true" ] || [ -n "${LOGS_CONFIG}" ] ; then
       mkdir dist/conf.d
     fi
 
     # add checks configs
-    if [ "$DD_ENABLE_CHECKS" = "true" ]; then
+    if [ "${DD_ENABLE_CHECKS}" = "true" ]; then
       mv conf.d/* dist/conf.d
     fi
 
     # add logs configs
-    if [ -n "$LOGS_CONFIG" ]; then
-      mkdir -p $LOGS_CONFIG_DIR
-      ruby scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.1.log"
+    if [ -n "${LOGS_CONFIG}" ]; then
+      mkdir -p ${LOGS_CONFIG_DIR}
+      ruby scripts/create_logs_config.rb 2>&1 | tee -a "${DATADOG_DIR}/ruby_script.1.log"
     fi
 
     # The yaml file requires the tags to be an array,
@@ -47,48 +47,48 @@ setup_datadog() {
     # so they must be grabbed separately
     sed -i "s~log_file: TRACE_LOG_FILE~log_file: ${DATADOG_DIR}/trace.log~" dist/datadog.yaml
 
-    if [ -n "$DD_SKIP_SSL_VALIDATION" ]; then
+    if [ -n "${DD_SKIP_SSL_VALIDATION}" ]; then
       sed -i "s~# skip_ssl_validation: no~skip_ssl_validation: yes~" dist/datadog.yaml
     fi
 
     # set logs, traces and metrics hostname to the VM hostname
-    if [ "$DD_ENABLE_CHECKS" != "true" ]; then
+    if [ "${DD_ENABLE_CHECKS}" != "true" ]; then
       sed -i "s~# enable_metadata_collection: true~enable_metadata_collection: false~" dist/datadog.yaml
-      host "$CF_INSTANCE_IP"
+      host "${CF_INSTANCE_IP}"
       if [ $? -eq 0 ]; then
-          IFS=. read -a VM_HOSTNAME <<< $(host $CF_INSTANCE_IP | awk '{print $5}')
-          sed -i "s~# hostname: mymachine.mydomain~hostname: $VM_HOSTNAME~" dist/datadog.yaml
+          IFS=. read -a VM_HOSTNAME <<< $(host ${CF_INSTANCE_IP} | awk '{print $5}')
+          sed -i "s~# hostname: mymachine.mydomain~hostname: ${VM_HOSTNAME}~" dist/datadog.yaml
       fi
     else
       sed -i "s~# hostname: mymachine.mydomain~hostname: $(hostname)~" dist/datadog.yaml
     fi
 
-    if [ -n "$DD_HTTP_PROXY" ]; then
+    if [ -n "${DD_HTTP_PROXY}" ]; then
       sed -i "s~# proxy:~proxy:~" dist/datadog.yaml
-      sed -i "s~#   http: HTTP_PROXY~  http: $DD_HTTP_PROXY~" dist/datadog.yaml
+      sed -i "s~#   http: HTTP_PROXY~  http: ${DD_HTTP_PROXY}~" dist/datadog.yaml
     else
-      if [ -n "$HTTP_PROXY" ]; then
+      if [ -n "${HTTP_PROXY}" ]; then
         sed -i "s~# proxy:~proxy:~" dist/datadog.yaml
-        sed -i "s~#   http: HTTP_PROXY~  http: $HTTP_PROXY~" dist/datadog.yaml
+        sed -i "s~#   http: HTTP_PROXY~  http: ${HTTP_PROXY}~" dist/datadog.yaml
       fi
     fi
-    if [ -n "$DD_HTTPS_PROXY" ]; then
+    if [ -n "${DD_HTTPS_PROXY}" ]; then
       sed -i "s~# proxy:~proxy:~" dist/datadog.yaml
-      sed -i "s~#   https: HTTPS_PROXY~  https: $DD_HTTPS_PROXY~" dist/datadog.yaml
+      sed -i "s~#   https: HTTPS_PROXY~  https: ${DD_HTTPS_PROXY}~" dist/datadog.yaml
     else
-      if [ -n "$HTTPS_PROXY" ]; then
+      if [ -n "${HTTPS_PROXY}" ]; then
         sed -i "s~# proxy:~proxy:~" dist/datadog.yaml
-        sed -i "s~#   https: HTTPS_PROXY~  https: $HTTPS_PROXY~" dist/datadog.yaml
+        sed -i "s~#   https: HTTPS_PROXY~  https: ${HTTPS_PROXY}~" dist/datadog.yaml
       fi
     fi
 
     #Override default EXPVAR Port
-    if [ -n "$DD_EXPVAR_PORT" ]; then
-      sed -i "s~# expvar_port: 5000~expvar_port: $DD_EXPVAR_PORT~" dist/datadog.yaml
+    if [ -n "${DD_EXPVAR_PORT}" ]; then
+      sed -i "s~# expvar_port: 5000~expvar_port: ${DD_EXPVAR_PORT}~" dist/datadog.yaml
     fi
     #Override default CMD Port
-    if [ -n "$DD_CMD_PORT" ]; then
-      sed -i "s~# cmd_port: 5001~cmd_port: $DD_CMD_PORT~" dist/datadog.yaml
+    if [ -n "${DD_CMD_PORT}" ]; then
+      sed -i "s~# cmd_port: 5001~cmd_port: ${DD_CMD_PORT}~" dist/datadog.yaml
     fi
 
     # Create folder for storing PID files
@@ -97,17 +97,17 @@ setup_datadog() {
     # DSD requires its own config file
     cp dist/datadog.yaml dist/dogstatsd.yaml
 
-    if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
-      if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
+    if [ -a ./agent ] && { [ "${DD_LOGS_ENABLED}" = "true" ] || [ "${DD_ENABLE_CHECKS}" = "true" ]; }; then
+      if [ "${DD_LOGS_ENABLED}" = "true" -a "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
         echo "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=${DATADOG_DIR}/agent.log
         export DD_IOT_HOST=false
-        sed -i "s~log_file: AGENT_LOG_FILE~log_file: $DD_LOG_FILE~" dist/datadog.yaml
+        sed -i "s~log_file: AGENT_LOG_FILE~log_file: ${DD_LOG_FILE}~" dist/datadog.yaml
       fi
     else
       export DD_LOG_FILE=${DATADOG_DIR}/dogstatsd.log
-      sed -i "s~log_file: AGENT_LOG_FILE~log_file: $DD_LOG_FILE~" dist/datadog.yaml
+      sed -i "s~log_file: AGENT_LOG_FILE~log_file: ${DD_LOG_FILE}~" dist/datadog.yaml
     fi
   popd
 }
@@ -136,17 +136,17 @@ start_datadog() {
       fi
     fi
 
-    if [ -a ./agent ] && { [ "$DD_LOGS_ENABLED" = "true" ] || [ "$DD_ENABLE_CHECKS" = "true" ]; }; then
-      if [ "$DD_LOGS_ENABLED" = "true" -a "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
+    if [ -a ./agent ] && { [ "${DD_LOGS_ENABLED}" = "true" ] || [ "${DD_ENABLE_CHECKS}" = "true" ]; }; then
+      if [ "${DD_LOGS_ENABLED}" = "true" -a "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
         echo "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
         echo "Starting Datadog agent"
-        ruby scripts/create_logs_config.rb 2>&1 | tee -a "$DATADOG_DIR/ruby_script.4.log"
+        ruby scripts/create_logs_config.rb 2>&1 | tee -a "${DATADOG_DIR}/ruby_script.4.log"
 
-        if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
+        if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
         else
           ./agent run --cfgpath dist/ --pidfile run/agent.pid &
@@ -155,7 +155,7 @@ start_datadog() {
     else
       echo "Starting dogstatsd agent"
       export DD_LOG_FILE=dogstatsd.log
-      if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
+      if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
         ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
       else
         ./dogstatsd start --cfgpath dist/ &
@@ -163,7 +163,7 @@ start_datadog() {
       echo $! > run/dogstatsd.pid
     fi
     echo "Starting trace agent"
-    if [ "$SUPPRESS_DD_AGENT_OUTPUT" = "true" ]; then
+    if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
     else
       ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid &
@@ -179,7 +179,7 @@ stop_datadog() {
       # first try to stop the agent so we don't lose data and then force it
       (./agent stop --cfgpath dist/) || true
       agent_command="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-      find_pid_kill_and_wait "$agent_command" "${DATADOG_DIR}/run/agent.pid" 5 1 || true
+      find_pid_kill_and_wait "${agent_command}" "${DATADOG_DIR}/run/agent.pid" 5 1 || true
       kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
       rm -f "run/agent.pid"
     fi
@@ -224,10 +224,10 @@ monit_datadog() {
 }
 
 main() {
-  if [ -z "$DD_API_KEY" ]; then
+  if [ -z "${DD_API_KEY}" ]; then
     echo "Datadog API Key not set, not starting Datadog"
   else
-    exec 9> "$LOCKFILE" || exit 1
+    exec 9> "${LOCKFILE}" || exit 1
     if flock -x -n 9; then
       echo "starting datadog"
       start_datadog
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 5173f21..40c2a3a 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -54,7 +54,7 @@ main() {
 
     # update logs configs with the new tags
     log_message "$0" "$$" "running create_logs_config ruby script"
-    ruby "${DATADOG_DIR}/scripts/create_logs_config.rb" 2>&1 | tee -a "$DATADOG_DIR/ruby_script.3.log"
+    ruby "${DATADOG_DIR}/scripts/create_logs_config.rb" 2>&1 | tee -a "${DATADOG_DIR}/ruby_script.3.log"
 
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
@@ -78,4 +78,4 @@ main() {
 }
 
 # for debugging purposes
-main "$@" 2>&1 | tee -a "$DEBUG_FILE"
+main "$@" 2>&1 | tee -a "${DEBUG_FILE}"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 9ff25fa..55cc9fb 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -128,22 +128,22 @@ find_pid_kill_and_wait() {
 # redirect forwards all standard inputs to a TCP socket listening on port STD_LOG_COLLECTION_PORT.
 redirect() {
   while kill -0 $$ 2>/dev/null; do
-    if [ "$DD_SPARSE_APP_LOGS" = "true" ]; then
-        python "${DATADOG_DIR}/scripts/nc.py" "$STD_LOG_COLLECTION_PORT" || sleep 0.5
+    if [ "${DD_SPARSE_APP_LOGS}" = "true" ]; then
+        python "${DATADOG_DIR}/scripts/nc.py" "${STD_LOG_COLLECTION_PORT}" || sleep 0.5
     else
-        nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
+        nc localhost "${STD_LOG_COLLECTION_PORT}" || sleep 0.5
     fi
     log_message "$0" "$$" "Resetting buildpack log redirection"
-    if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
-      HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
+    if [ "${DD_DEBUG_STD_REDIRECTION}" = "true" ]; then
+      HTTP_PROXY=${DD_HTTP_PROXY} HTTPS_PROXY=${DD_HTTPS_PROXY} NO_PROXY=${DD_NO_PROXY} curl \
       -X POST -H "Content-type: application/json" \
       -d "{
             \"title\": \"Resetting buildpack log redirection\",
-            \"text\": \"TCP socket on port $STD_LOG_COLLECTION_PORT for log redirection closed. Restarting it.\",
+            \"text\": \"TCP socket on port ${STD_LOG_COLLECTION_PORT} for log redirection closed. Restarting it.\",
             \"priority\": \"normal\",
             \"tags\": $(python ${DATADOG_DIR}/scripts/get_tags.py),
             \"alert_type\": \"info\"
-      }" "${DD_API_SITE}v1/events?api_key=$DD_API_KEY"
+      }" "${DD_API_SITE}v1/events?api_key=${DD_API_KEY}"
     fi
   done
 }
diff --git a/lib/test-endpoint.sh b/lib/test-endpoint.sh
index d561a6e..f482b85 100644
--- a/lib/test-endpoint.sh
+++ b/lib/test-endpoint.sh
@@ -8,16 +8,16 @@ unset DD_LOGS_VALID_ENDPOINT
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 DD_EU_API_SITE="https://api.datadoghq.eu/api/"
 DD_US_API_SITE="https://api.datadoghq.com/api/"
-DD_API_SITE=$DD_US_API_SITE
+DD_API_SITE=${DD_US_API_SITE}
 DD_USE_EU=false
 
 # Sets variable in order of DD_PROXY_HTTP -> DD_HTTP_PROXY -> HTTP_PROXY
-DD_PROXY_HTTP_VAR=${DD_PROXY_HTTP:-$DD_HTTP_PROXY}
-DD_PROXY_HTTP_VAR=${DD_PROXY_HTTP_VAR:-$HTTP_PROXY}
+DD_PROXY_HTTP_VAR=${DD_PROXY_HTTP:-${DD_HTTP_PROXY}}
+DD_PROXY_HTTP_VAR=${DD_PROXY_HTTP_VAR:-${HTTP_PROXY}}
 
 # Sets variable in order of DD_PROXY_HTTPS -> DD_HTTPS_PROXY -> HTTPS_PROXY
-DD_PROXY_HTTPS_VAR=${DD_PROXY_HTTPS:-$DD_HTTPS_PROXY}
-DD_PROXY_HTTPS_VAR=${DD_PROXY_HTTPS_VAR:-$HTTPS_PROXY}
+DD_PROXY_HTTPS_VAR=${DD_PROXY_HTTPS:-${DD_HTTPS_PROXY}}
+DD_PROXY_HTTPS_VAR=${DD_PROXY_HTTPS_VAR:-${HTTPS_PROXY}}
 
 DD_STRIPPED_PROXY_HTTPS="${DD_PROXY_HTTPS_VAR//https:\/\/}"
 DD_STRIPPED_PROXY_HTTPS="${DD_STRIPPED_PROXY_HTTPS//http:\/\/}"
@@ -28,64 +28,64 @@ DD_DEFAULT_HTTPS_US_ENDPOINT="agent-http-intake.logs.datadoghq.com:443"
 DD_DEFAULT_TCP_EU_ENDPOINT="agent-intake.logs.datadoghq.eu:443"
 DD_DEFAULT_TCP_US_ENDPOINT="agent-intake.logs.datadoghq.com:10516"
 
-if [ "$DD_SITE" = "datadoghq.eu" ]; then
+if [ "${DD_SITE}" = "datadoghq.eu" ]; then
   DD_USE_EU=true
-  DD_API_SITE=$DD_EU_API_SITE
+  DD_API_SITE=${DD_EU_API_SITE}
 fi
 
-if [ "$DD_LOGS_CONFIG_USE_HTTP" = true ]; then
-  if [ -n "$DD_STRIPPED_PROXY_HTTPS" ]; then
-    DEFAULT_LOGS_ENDPOINT="$DD_STRIPPED_PROXY_HTTPS"
+if [ "${DD_LOGS_CONFIG_USE_HTTP}" = true ]; then
+  if [ -n "${DD_STRIPPED_PROXY_HTTPS}" ]; then
+    DEFAULT_LOGS_ENDPOINT="${DD_STRIPPED_PROXY_HTTPS}"
   else
-    if [ "$DD_USE_EU" = true ]; then
-      DEFAULT_LOGS_ENDPOINT="$DD_DEFAULT_HTTPS_EU_ENDPOINT"
+    if [ "${DD_USE_EU}" = true ]; then
+      DEFAULT_LOGS_ENDPOINT="${DD_DEFAULT_HTTPS_EU_ENDPOINT}"
     else
-      DEFAULT_LOGS_ENDPOINT="$DD_DEFAULT_HTTPS_US_ENDPOINT"
+      DEFAULT_LOGS_ENDPOINT="${DD_DEFAULT_HTTPS_US_ENDPOINT}"
     fi
   fi
 else
-  if [ "$DD_USE_EU" = true ]; then
-    DEFAULT_LOGS_ENDPOINT="$DD_DEFAULT_TCP_EU_ENDPOINT"
+  if [ "${DD_USE_EU}" = true ]; then
+    DEFAULT_LOGS_ENDPOINT="${DD_DEFAULT_TCP_EU_ENDPOINT}"
   else
-    DEFAULT_LOGS_ENDPOINT="$DD_DEFAULT_TCP_US_ENDPOINT"
+    DEFAULT_LOGS_ENDPOINT="${DD_DEFAULT_TCP_US_ENDPOINT}"
   fi
 fi
 
-if [ -z "$DD_LOGS_CONFIG_LOGS_DD_URL" ]; then
+if [ -z "${DD_LOGS_CONFIG_LOGS_DD_URL}" ]; then
   # Initialize to default value based on the following order:
   # 1) If both the host/port for logs is specified, use that
   # 2) If the DD_PROXY_HTTPS is set, use that
   # 3) If DD_SITE is set to datadoghq.eu, use default EU host/port
   # 4) Default back to US logs host/port combo.
-  if [ -n "$DD_LOGS_CONFIG_DD_PORT" ] && [ -n "$DD_LOGS_CONFIG_DD_URL" ]; then
-    DD_LOGS_CONFIG_LOGS_DD_URL="$DD_LOGS_CONFIG_DD_URL:$DD_LOGS_CONFIG_DD_PORT"
+  if [ -n "${DD_LOGS_CONFIG_DD_PORT}" ] && [ -n "${DD_LOGS_CONFIG_DD_URL}" ]; then
+    DD_LOGS_CONFIG_LOGS_DD_URL="${DD_LOGS_CONFIG_DD_URL}:${DD_LOGS_CONFIG_DD_PORT}"
   else
-    DD_LOGS_CONFIG_LOGS_DD_URL="$DEFAULT_LOGS_ENDPOINT"
+    DD_LOGS_CONFIG_LOGS_DD_URL="${DEFAULT_LOGS_ENDPOINT}"
   fi
 fi
 
-if [ "${DD_LOGS_ENABLED}" = "true" -a -n $DD_LOGS_CONFIG_LOGS_DD_URL -a "$DD_SKIP_LOGS_TEST" != "true" ]; then
-  echo "Validating log endpoint $DD_LOGS_CONFIG_LOGS_DD_URL"
-  LOGS_ENDPOINT=`echo $DD_LOGS_CONFIG_LOGS_DD_URL | cut -d ":" -f1`
-  LOGS_PORT=`echo $DD_LOGS_CONFIG_LOGS_DD_URL | cut -d ":" -f2`
+if [ "${DD_LOGS_ENABLED}" = "true" -a -n ${DD_LOGS_CONFIG_LOGS_DD_URL} -a "${DD_SKIP_LOGS_TEST}" != "true" ]; then
+  echo "Validating log endpoint ${DD_LOGS_CONFIG_LOGS_DD_URL}"
+  LOGS_ENDPOINT=`echo ${DD_LOGS_CONFIG_LOGS_DD_URL} | cut -d ":" -f1`
+  LOGS_PORT=`echo ${DD_LOGS_CONFIG_LOGS_DD_URL} | cut -d ":" -f2`
 
   # Try establishing tcp connection to logs endpoint with 5s timeout
-  nc -w5 $LOGS_ENDPOINT $LOGS_PORT < /dev/null
+  nc -w5 ${LOGS_ENDPOINT} ${LOGS_PORT} < /dev/null
 
   # Check out exit code and export a variable for subsequent scripts to use
   if [ $? -ne 0 ]; then
     export DD_LOGS_VALID_ENDPOINT="false"
-    echo "Could not establish a connection to $DD_LOGS_CONFIG_LOGS_DD_URL."
+    echo "Could not establish a connection to ${DD_LOGS_CONFIG_LOGS_DD_URL}."
     # Post alert to datadog
-    HTTP_PROXY=$DD_PROXY_HTTP_VAR HTTPS_PROXY=$DD_PROXY_HTTPS_VAR NO_PROXY=$DD_NO_PROXY curl \
+    HTTP_PROXY=${DD_PROXY_HTTP_VAR} HTTPS_PROXY=${DD_PROXY_HTTPS_VAR} NO_PROXY=${DD_NO_PROXY} curl \
       -X POST -H "Content-type: application/json" \
       -d "{
             \"title\": \"Log endpoint cannot be reached - Log collection not started\",
-            \"text\": \"Could not establish a connection to $DD_LOGS_CONFIG_LOGS_DD_URL after 5 seconds. Log collection has not been started.\",
+            \"text\": \"Could not establish a connection to ${DD_LOGS_CONFIG_LOGS_DD_URL} after 5 seconds. Log collection has not been started.\",
             \"priority\": \"normal\",
             \"tags\": $(python ${DATADOG_DIR}/scripts/get_tags.py),
             \"alert_type\": \"error\"
-      }" "${DD_API_SITE}v1/events?api_key=$DD_API_KEY"
+      }" "${DD_API_SITE}v1/events?api_key=${DD_API_KEY}"
   else
     export DD_LOGS_VALID_ENDPOINT="true"
   fi

From 36d864514df9bcdaeaca44ae8fe60ec25674bd07 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 16 Dec 2022 22:57:51 +0100
Subject: [PATCH 21/54] small cleanup

---
 lib/run-datadog.sh   |  4 +---
 lib/scripts/utils.sh |  4 ++--
 lib/test-endpoint.sh | 12 +++++-------
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 6959d77..54ace31 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -13,8 +13,6 @@ source ${DATADOG_DIR}/scripts/utils.sh
 
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 
-
-
 setup_datadog() {
   pushd "${DATADOG_DIR}"
 
@@ -137,7 +135,7 @@ start_datadog() {
     fi
 
     if [ -a ./agent ] && { [ "${DD_LOGS_ENABLED}" = "true" ] || [ "${DD_ENABLE_CHECKS}" = "true" ]; }; then
-      if [ "${DD_LOGS_ENABLED}" = "true" -a "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
+      if [ "${DD_LOGS_ENABLED}" = "true" ] && [ "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
         echo "Log endpoint not valid, not starting agent"
       else
         export DD_LOG_FILE=agent.log
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 55cc9fb..21cd682 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -40,14 +40,14 @@ wait_pid() {
   local countdown=$(( 100 )) # temporary to workaround a /bin/dash syntax error
   local ps_out="$(ps ax | grep ${pid} | grep -v grep)"
 
-  if [ -e "/proc/${pid}" -o -n "${ps_out}" ]; then
+  if [ -e "/proc/${pid}" ] || [ -n "${ps_out}" ]; then
     if [ "${try_kill}" = "1" ]; then
       log_message "$0" "Killing ${pidfile}: ${pid}"
       kill "${pid}"
     fi
     while [ -e "/proc/${pid}" ]; do
       sleep 0.1
-      [ "${countdown}" != '0' -a $(( "${countdown}" % 10 )) = '0' ] && echo -n .
+      [ "${countdown}" != '0' ] && [ $(( "${countdown}" % 10 )) = '0' ] && echo -n .
       if [ "${timeout}" -gt 0 ]; then
         if [ "${countdown}" -eq 0 ]; then
           if [ "${force}" = "1" ]; then
diff --git a/lib/test-endpoint.sh b/lib/test-endpoint.sh
index f482b85..da908b4 100644
--- a/lib/test-endpoint.sh
+++ b/lib/test-endpoint.sh
@@ -64,16 +64,14 @@ if [ -z "${DD_LOGS_CONFIG_LOGS_DD_URL}" ]; then
   fi
 fi
 
-if [ "${DD_LOGS_ENABLED}" = "true" -a -n ${DD_LOGS_CONFIG_LOGS_DD_URL} -a "${DD_SKIP_LOGS_TEST}" != "true" ]; then
+if [ "${DD_LOGS_ENABLED}" = "true" ] && [ -n "${DD_LOGS_CONFIG_LOGS_DD_URL}" ] && [ "${DD_SKIP_LOGS_TEST}" != "true" ]; then
   echo "Validating log endpoint ${DD_LOGS_CONFIG_LOGS_DD_URL}"
-  LOGS_ENDPOINT=`echo ${DD_LOGS_CONFIG_LOGS_DD_URL} | cut -d ":" -f1`
-  LOGS_PORT=`echo ${DD_LOGS_CONFIG_LOGS_DD_URL} | cut -d ":" -f2`
+  LOGS_ENDPOINT=$(echo "${DD_LOGS_CONFIG_LOGS_DD_URL}" | cut -d ":" -f1)
+  LOGS_PORT=$(echo "${DD_LOGS_CONFIG_LOGS_DD_URL}" | cut -d ":" -f2)
 
   # Try establishing tcp connection to logs endpoint with 5s timeout
-  nc -w5 ${LOGS_ENDPOINT} ${LOGS_PORT} < /dev/null
-
   # Check out exit code and export a variable for subsequent scripts to use
-  if [ $? -ne 0 ]; then
+  if ! nc -w5 "${LOGS_ENDPOINT}" "${LOGS_PORT}" < /dev/null; then
     export DD_LOGS_VALID_ENDPOINT="false"
     echo "Could not establish a connection to ${DD_LOGS_CONFIG_LOGS_DD_URL}."
     # Post alert to datadog
@@ -83,7 +81,7 @@ if [ "${DD_LOGS_ENABLED}" = "true" -a -n ${DD_LOGS_CONFIG_LOGS_DD_URL} -a "${DD_
             \"title\": \"Log endpoint cannot be reached - Log collection not started\",
             \"text\": \"Could not establish a connection to ${DD_LOGS_CONFIG_LOGS_DD_URL} after 5 seconds. Log collection has not been started.\",
             \"priority\": \"normal\",
-            \"tags\": $(python ${DATADOG_DIR}/scripts/get_tags.py),
+            \"tags\": $(python "${DATADOG_DIR}"/scripts/get_tags.py),
             \"alert_type\": \"error\"
       }" "${DD_API_SITE}v1/events?api_key=${DD_API_KEY}"
   else

From dbb496e8b8aa81435eeefde59caae07c113d82e9 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 12:29:20 +0100
Subject: [PATCH 22/54] use curly braces for env vars access everywhere

---
 bin/compile |  6 ++--
 bin/supply  |  7 +++--
 build       | 88 ++++++++++++++++++++++++++---------------------------
 3 files changed, 51 insertions(+), 50 deletions(-)

diff --git a/bin/compile b/bin/compile
index 194aca6..0e254ba 100755
--- a/bin/compile
+++ b/bin/compile
@@ -4,7 +4,7 @@ echo "-----> DatadogBuildpack/compile"
 
 
 BIN_DIR=$(cd $(dirname $0); pwd)
-ROOT_DIR=$(dirname $BIN_DIR)
+ROOT_DIR=$(dirname ${BIN_DIR})
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
@@ -64,7 +64,7 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
 # Export all environment variables to a file that scripts can use later
 # We need to handle quotes and comments; sed command found in https://gist.github.com/mihow/9c7f559807069a03e302605691f85572
-printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$BUILD_DIR/.datadog/.raw_datadog_env"
+printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
-python "$DATADOG_DIR/scripts/parse_env_vars.py" "$DATADOG_DIR/.raw_datadog_env" "$DATADOG_DIR/.datadog_env"
\ No newline at end of file
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index ad636cd..857a0e5 100755
--- a/bin/supply
+++ b/bin/supply
@@ -4,7 +4,7 @@ echo "-----> DatadogBuildpack/supply"
 
 
 BIN_DIR=$(cd $(dirname $0); pwd)
-ROOT_DIR=$(dirname $BIN_DIR)
+ROOT_DIR=$(dirname ${BIN_DIR})
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
@@ -33,6 +33,7 @@ cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
@@ -64,8 +65,8 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
 # Export all environment variables to a file that scripts can use later
 # We need to handle quotes and comments; sed command found in https://gist.github.com/mihow/9c7f559807069a03e302605691f85572
-printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$BUILD_DIR/.datadog/.raw_datadog_env"
+printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
-python "$DATADOG_DIR/scripts/parse_env_vars.py" "$DATADOG_DIR/.raw_datadog_env" "$DATADOG_DIR/.datadog_env"
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
 
diff --git a/build b/build
index c80d8f1..fe7c7c1 100755
--- a/build
+++ b/build
@@ -3,89 +3,89 @@ set -e
 
 SRCDIR=$(cd "$(dirname $0)/." && pwd)
 NAME="datadog-cloudfoundry-buildpack"
-ZIPFILE="$NAME.zip"
+ZIPFILE="${NAME}.zip"
 DOWNLOAD_BASE_URL="https://s3.amazonaws.com/apt.datadoghq.com/pool/d/da/datadog-"
-TRACEAGENT_DOWNLOAD_URL=$DOWNLOAD_BASE_URL"agent_"
-IOT_AGENT_DOWNLOAD_URL=$DOWNLOAD_BASE_URL"iot-agent_"
-DOGSTATSD_DOWNLOAD_URL=$DOWNLOAD_BASE_URL"dogstatsd_"
+TRACEAGENT_DOWNLOAD_URL=${DOWNLOAD_BASE_URL}"agent_"
+IOT_AGENT_DOWNLOAD_URL=${DOWNLOAD_BASE_URL}"iot-agent_"
+DOGSTATSD_DOWNLOAD_URL=${DOWNLOAD_BASE_URL}"dogstatsd_"
 DOWNLOAD_URL_TAIL="-1_amd64.deb"
 AGENT_DEFAULT_VERSION="7.40.1"
 
-TMPDIR="$SRCDIR/tmp"
+TMPDIR="${SRCDIR}/tmp"
 
 function download_trace_agent() {
-  local trace_version="${1:-$AGENT_DEFAULT_VERSION}"
-  local trace_agent_download_url="$TRACEAGENT_DOWNLOAD_URL$trace_version$DOWNLOAD_URL_TAIL"
+  local trace_version="${1:-${AGENT_DEFAULT_VERSION}}"
+  local trace_agent_download_url="${TRACEAGENT_DOWNLOAD_URL}${trace_version}${DOWNLOAD_URL_TAIL}"
 
-  mkdir -p $TMPDIR
-  curl -L $trace_agent_download_url -o ./tmp/datadog-agent.deb
-  pushd $TMPDIR
+  mkdir -p ${TMPDIR}
+  curl -L ${trace_agent_download_url} -o ./tmp/datadog-agent.deb
+  pushd ${TMPDIR}
     dpkg -x datadog-agent.deb .
   popd
-  cp $TMPDIR/opt/datadog-agent/embedded/bin/trace-agent $SRCDIR/lib/trace-agent
-  rm -rf $TMPDIR/*
+  cp ${TMPDIR}/opt/datadog-agent/embedded/bin/trace-agent ${SRCDIR}/lib/trace-agent
+  rm -rf ${TMPDIR}/*
 }
 
 function download_iot_agent() {
-  local iot_version="${1:-$AGENT_DEFAULT_VERSION}"
-  local iot_agent_download_url="$IOT_AGENT_DOWNLOAD_URL$iot_version$DOWNLOAD_URL_TAIL"
+  local iot_version="${1:-${AGENT_DEFAULT_VERSION}}"
+  local iot_agent_download_url="${IOT_AGENT_DOWNLOAD_URL}${iot_version}${DOWNLOAD_URL_TAIL}"
 
-  mkdir -p $TMPDIR
-  curl -L $iot_agent_download_url -o ./tmp/datadog-agent.deb
-  pushd $TMPDIR
+  mkdir -p ${TMPDIR}
+  curl -L ${iot_agent_download_url} -o ./tmp/datadog-agent.deb
+  pushd ${TMPDIR}
     dpkg -x datadog-agent.deb .
   popd
-  cp $TMPDIR/opt/datadog-agent/bin/agent/agent $SRCDIR/lib/agent
-  rm -rf $TMPDIR/*
+  cp ${TMPDIR}/opt/datadog-agent/bin/agent/agent ${SRCDIR}/lib/agent
+  rm -rf ${TMPDIR}/*
 }
 
 function download_dogstatsd() {
-  local dogstatsd_version="${1:-$AGENT_DEFAULT_VERSION}"
-  local dogstatsd_download_url="$DOGSTATSD_DOWNLOAD_URL$dogstatsd_version$DOWNLOAD_URL_TAIL"
+  local dogstatsd_version="${1:-${AGENT_DEFAULT_VERSION}}"
+  local dogstatsd_download_url="${DOGSTATSD_DOWNLOAD_URL}${dogstatsd_version}${DOWNLOAD_URL_TAIL}"
 
-  mkdir -p $TMPDIR
-  curl -L $dogstatsd_download_url -o ./tmp/dogstatsd.deb
-  pushd $TMPDIR
+  mkdir -p ${TMPDIR}
+  curl -L ${dogstatsd_download_url} -o ./tmp/dogstatsd.deb
+  pushd ${TMPDIR}
     dpkg -x dogstatsd.deb .
   popd
-  cp $TMPDIR/opt/datadog-dogstatsd/bin/dogstatsd $SRCDIR/lib/dogstatsd
-  rm -rf $TMPDIR/*
+  cp ${TMPDIR}/opt/datadog-dogstatsd/bin/dogstatsd ${SRCDIR}/lib/dogstatsd
+  rm -rf ${TMPDIR}/*
 }
 
 function main() {
-  if [ ! -f $SRCDIR/lib/dogstatsd ] || [ ! -f $SRCDIR/lib/trace-agent ]; then
+  if [ ! -f ${SRCDIR}/lib/dogstatsd ] || [ ! -f ${SRCDIR}/lib/trace-agent ]; then
     DOWNLOAD="true"
   fi
-  if [ -n "$IOT_AGENT" ] && [ ! -f $SRCDIR/lib/agent ]; then
+  if [ -n "${IOT_AGENT}" ] && [ ! -f ${SRCDIR}/lib/agent ]; then
     DOWNLOAD="true"
   fi
-  if [ -n "$REFRESH_ASSETS" ]; then
+  if [ -n "${REFRESH_ASSETS}" ]; then
     DOWNLOAD="true"
   fi
-  if [ -n "$DOWNLOAD" ]; then
+  if [ -n "${DOWNLOAD}" ]; then
     # Delete the old ones
-    rm -f $SRCDIR/lib/agent
-    rm -f $SRCDIR/lib/dogstatsd
-    rm -f $SRCDIR/lib/trace-agent
+    rm -f ${SRCDIR}/lib/agent
+    rm -f ${SRCDIR}/lib/dogstatsd
+    rm -f ${SRCDIR}/lib/trace-agent
 
     # Download the new ones
-    VERSION=${VERSION:-$AGENT_DEFAULT_VERSION}
+    VERSION=${VERSION:-${AGENT_DEFAULT_VERSION}}
 
-    download_trace_agent $VERSION
-    chmod +x $SRCDIR/lib/trace-agent
+    download_trace_agent ${VERSION}
+    chmod +x ${SRCDIR}/lib/trace-agent
 
-    download_iot_agent $VERSION
-    chmod +x $SRCDIR/lib/agent
+    download_iot_agent ${VERSION}
+    chmod +x ${SRCDIR}/lib/agent
 
-    download_dogstatsd $VERSION
-    chmod +x $SRCDIR/lib/dogstatsd
+    download_dogstatsd ${VERSION}
+    chmod +x ${SRCDIR}/lib/dogstatsd
   fi
 
-  rm -f $ZIPFILE
+  rm -f ${ZIPFILE}
 
-  pushd $SRCDIR
-    if [ ! "$NO_ZIP" ]; then
-      zip -r "$ZIPFILE" lib bin
+  pushd ${SRCDIR}
+    if [ ! "${NO_ZIP}" ]; then
+      zip -r "${ZIPFILE}" lib bin
     fi
   popd
 }

From 3fbf85efc1c2f930fe2a386cd4214f8548f8ee4b Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 12:30:03 +0100
Subject: [PATCH 23/54] add util function check_if_running + cleanup

---
 lib/run-datadog.sh                 | 45 +++++++++++++++---------------
 lib/scripts/check_datadog.sh       | 22 +++++++--------
 lib/scripts/create_logs_config.rb  |  4 ++-
 lib/scripts/update_agent_config.sh | 24 +++++++++-------
 lib/scripts/utils.sh               | 13 ++++++++-
 5 files changed, 62 insertions(+), 46 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 54ace31..2ef1a9f 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -37,7 +37,7 @@ setup_datadog() {
     # add logs configs
     if [ -n "${LOGS_CONFIG}" ]; then
       mkdir -p ${LOGS_CONFIG_DIR}
-      ruby scripts/create_logs_config.rb 2>&1 | tee -a "${DATADOG_DIR}/ruby_script.1.log"
+      ruby scripts/create_logs_config.rb
     fi
 
     # The yaml file requires the tags to be an array,
@@ -141,9 +141,13 @@ start_datadog() {
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
-        echo "Starting Datadog agent"
-        ruby scripts/create_logs_config.rb 2>&1 | tee -a "${DATADOG_DIR}/ruby_script.4.log"
+        # update logs configs
+        if [ -n "${LOGS_CONFIG}" ]; then
+          mkdir -p "${LOGS_CONFIG_DIR}"
+          ruby scripts/create_logs_config.rb
+        fi
 
+        echo "Starting Datadog agent"
         if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
         else
@@ -171,36 +175,31 @@ start_datadog() {
 
 stop_datadog() {
   pushd "${DATADOG_DIR}"
-    # TODO: use a fallback approach in case the pid file is missing
-    if kill -0 "$(cat ${DATADOG_DIR}/run/agent.pid)" >/dev/null; then
-      echo "Stopping agent process, pid: $(cat run/agent.pid)"
+    if check_if_running "${AGENT_PIDFILE}" "${AGENT_CMD}"; then
+      echo "Stopping agent process, pid: $(cat "${AGENT_PIDFILE}")"
       # first try to stop the agent so we don't lose data and then force it
       (./agent stop --cfgpath dist/) || true
-      agent_command="./agent run --cfgpath dist/ --pidfile run/agent.pid"
-      find_pid_kill_and_wait "${agent_command}" "${DATADOG_DIR}/run/agent.pid" 5 1 || true
-      kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5 1
-      rm -f "run/agent.pid"
+      find_pid_kill_and_wait "${AGENT_CMD}" "${AGENT_PIDFILE}" 5 1 || true
+      kill_and_wait "${AGENT_PIDFILE}" 5 1
+      rm -f "${AGENT_PIDFILE}"
     fi
 
-    if kill -0 "$(cat "${DATADOG_DIR}/run/dogstatsd.pid")" > /dev/null; then
-      echo "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
-      dogstatsd_command="./dogstatsd start --cfgpath dist/"
-      kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
-      find_pid_kill_and_wait "${dogstatsd_command}" "${DATADOG_DIR}/run/dogstatsd.pid" 5 1 
-      rm -f "run/dogstatsd.pid"
+    if check_if_running "${DOGSTATSD_PIDFILE}" "${DOGSTATSD_CMD}"; then
+      echo "Stopping dogstatsd agent process, pid: $(cat "${DOGSTATSD_PIDFILE}")"
+      kill_and_wait "${DOGSTATSD_PIDFILE}" 5 1
+      find_pid_kill_and_wait "${DOGSTATSD_CMD}" "${DOGSTATSD_PIDFILE}" 5 1
+      rm -f "${DOGSTATSD_PIDFILE}"
     fi
 
-    if kill -0 "$(cat "${DATADOG_DIR}/run/trace-agent.pid")" >/dev/null; then
-      echo "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
-      trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
-      kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
-      find_pid_kill_and_wait "${trace_agent_command}" "${DATADOG_DIR}/run/trace-agent.pid" 5 1
-      rm -f "run/trace-agent.pid"
+     if check_if_running "${TRACE_AGENT_PIDFILE}" "${TRACE_AGENT_CMD}"; then
+      echo "Stopping trace agent process, pid: $(cat "${TRACE_AGENT_PIDFILE}")"
+      kill_and_wait "${TRACE_AGENT_PIDFILE}" 5 1
+      find_pid_kill_and_wait "${TRACE_AGENT_CMD}" "${TRACE_AGENT_PIDFILE}" 5 1
+      rm -f "${TRACE_AGENT_PIDFILE}"
     fi
   popd
 }
 
-
 monit_datadog() {
   while true; do
     if ! kill -0 $$; then
diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index 7fb2847..e2fb4d6 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -11,19 +11,19 @@ source "${DATADOG_DIR}/scripts/utils.sh"
 check_datadog() {
   while true; do
     log_message "$0" "$$" "Waiting for agent or dogstatsd process to start"
-    if kill -0 "$(cat "${DATADOG_DIR}/run/agent.pid")" > /dev/null; then
-        log_message "$0" "$$" "Found agent process"
-        if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
-          log_message "$0" "$$" "Found agent token"
-          break
-        else 
-          log_message "$0" "$$" "Agent token not found"
-        fi
+    if check_if_running "${AGENT_PIDFILE}" "${AGENT_CMD}"; then
+      log_message "$0" "$$" "found agent process: $(cat "${AGENT_PIDFILE}")"
+      if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
+        log_message "$0" "$$" "found agent token"
+        break
+      else 
+        log_message "$0" "$$" "agent token not found"
+      fi
     fi
 
-    if kill -0 "$(cat "${DATADOG_DIR}/run/dogstatsd.pid")" > /dev/null; then
-        log_message "$0" "$$" "Found dogstatsd process"
-        break
+    if check_if_running "${DOGSTATSD_PIDFILE}" "${DOGSTATSD_CMD}"; then
+      log_message "$0" "$$" "found dogstatsd process: $(cat "${DOGSTATSD_PIDFILE}")"
+      break
     fi
     sleep 1
   done
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index f6a045f..16cf730 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -1,6 +1,8 @@
 # Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2017-Present Datadog, Inc.
+# Copyright 2022-Present Datadog, Inc.
+
+#!/usr/bin/env ruby
 
 require 'json'
 
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 40c2a3a..87a3e93 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -4,8 +4,6 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2022-Present Datadog, Inc.
 
-
-
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 LOCK="${DATADOG_DIR}/update.lock"
 
@@ -13,7 +11,7 @@ LOCK="${DATADOG_DIR}/update.lock"
 . "${DATADOG_DIR}/scripts/utils.sh"
 
 release_lock() {
-    log_message "$0" "$$" "releasing LOCK ${LOCK}"
+    log_message "$0" "$$" "releasing lock '${LOCK}'"
     rmdir "${LOCK}"
 }
 
@@ -27,23 +25,30 @@ main() {
         sleep 1
     done
 
-    log_message "$0" "$$" "acquired LOCK ${LOCK}"
+    log_message "$0" "$$" "acquired lock '${LOCK}'"
    
     # ensures the lock is released on exit
     trap release_lock INT TERM EXIT
 
     # wait for the buildpack scripts to finish
     log_message "$0" "$$" "starting check_datadog script"
+
+    while ! [ -f "${DATADOG_DIR}/scripts/check_datadog.sh" ]; do
+        log_message "$0" "$$" "check_datadog.sh script not found, waiting..."
+        sleep 2
+    done
+
     timeout 300s "${DATADOG_DIR}/scripts/check_datadog.sh" 
     exit_code=$?
-    log_message "$0" "$$" "finished check_datadog script"
-
-    # verify that check_datadog didn't end because of the timeout
+    
+    # verify that check_datadog exited successfully 
     if [ ${exit_code} -ne  0 ]; then
         log_message "$0" "$$" "could not find agent, aborting update script!"
         exit ${exit_code}
     fi
 
+    log_message "$0" "$$" "finished check_datadog script"
+
     # source relevant DD tags
     . "${DATADOG_DIR}/.datadog_env"
 
@@ -53,8 +58,8 @@ main() {
     export LOGS_CONFIG
 
     # update logs configs with the new tags
-    log_message "$0" "$$" "running create_logs_config ruby script"
-    ruby "${DATADOG_DIR}/scripts/create_logs_config.rb" 2>&1 | tee -a "${DATADOG_DIR}/ruby_script.3.log"
+    log_message "$0" "$$" "Creating logs config"
+    ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
 
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
@@ -65,7 +70,6 @@ main() {
     log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
     log_message "$0" "$$" "(AFTER)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
 
-    
     # finishing up
     log_message "$0" "$$" "exporting .sourced_datadog_env file"
     printenv > "${DATADOG_DIR}/.sourced_datadog_env"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 21cd682..6e6a6a7 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -12,6 +12,17 @@ export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
 export LOGS_CONFIG
 
+export AGENT_PIDFILE="${DATADOG_DIR}/run/agent.pid"
+export AGENT_CMD="./agent run --cfgpath dist/ --pidfile run/agent.pid"
+
+export TRACE_AGENT_PIDFILE="${DATADOG_DIR}/run/trace-agent.pid"
+export TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
+
+export DOGSTATSD_PIDFILE="${DATADOG_DIR}/run/trace-agent.pid"
+export DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
+
+
+
 log_message() {
   local component="${1#/home/vcap/app/}"
   local pid="$2"
@@ -127,7 +138,7 @@ find_pid_kill_and_wait() {
 
 # redirect forwards all standard inputs to a TCP socket listening on port STD_LOG_COLLECTION_PORT.
 redirect() {
-  while kill -0 $$ 2>/dev/null; do
+  while kill -0 $$; do
     if [ "${DD_SPARSE_APP_LOGS}" = "true" ]; then
         python "${DATADOG_DIR}/scripts/nc.py" "${STD_LOG_COLLECTION_PORT}" || sleep 0.5
     else

From 4ed8cfd8aed3bc42bd2f2283c460a4238d220133 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 12:31:42 +0100
Subject: [PATCH 24/54] add update_tags ruby script

---
 bin/compile                        |  1 +
 lib/scripts/update_agent_config.sh |  4 ++-
 lib/scripts/update_tags.rb         | 45 ++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 lib/scripts/update_tags.rb

diff --git a/bin/compile b/bin/compile
index 0e254ba..19fdb81 100755
--- a/bin/compile
+++ b/bin/compile
@@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 87a3e93..f8d3b37 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -64,7 +64,9 @@ main() {
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
     log_message "$0" "$$" "Writing DD_TAGS to node_agent_tags.txt"
-    echo "${DD_TAGS}" | awk '{ printf "%s", $0 }' >  "${DATADOG_DIR}/node_agent_tags.txt"
+    
+    # update node_agent_tags.txt
+    ruby "${DATADOG_DIR}"/scripts/update_tags.rb
     
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
     log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
new file mode 100644
index 0000000..4b4dc54
--- /dev/null
+++ b/lib/scripts/update_tags.rb
@@ -0,0 +1,45 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022-Present Datadog, Inc.
+
+#!/usr/bin/env ruby
+
+# env vars
+DATADOG_DIR = ENV.fetch("DATADOG_DIR", "/home/vcap/app/.datadog")
+DD_TAGS = ENV.fetch("DD_TAGS", "")
+DD_NODE_AGENT_TAGS = ENV.fetch("DD_NODE_AGENT_TAGS", "")
+DD_UPDATE_SCRIPT_WARMUP = ENV.fetch("DD_UPDATE_SCRIPT_WARMUP", "180")
+
+# file paths
+timestamp_file = File.join(DATADOG_DIR, "startup_time")
+node_agent_tags_file = File.join(DATADOG_DIR, "node_agent_tags.txt")
+
+# read startup time set by the buildpack supply script
+timestamp = File.read(timestamp_file).strip
+time = Time.parse(timestamp)
+
+# storing all tags on this variable
+tags = []
+
+if ! DD_NODE_AGENT_TAGS.empty?
+    tags.concat(DD_NODE_AGENT_TAGS.split(','))
+
+if ! DD_TAGS.empty?
+    tags.concat(DD_TAGS.split(','))
+
+# if the script is executed during the warmup period, merge incoming tags with the existing tags
+# otherwise, override existing tags
+if Time.now - time <= DD_UPDATE_SCRIPT_WARMUP.to_i
+    if File.exists?(node_agent_tags_file)
+        node_tags = File.read(node_agent_tags_file).split(',')
+        tags.concat(node_tags)
+end
+
+# remove duplicates
+tags = tags.uniq
+
+# export tags
+File.write(node_agent_tags_file, tags.join(","))
+
+
+

From 58c59c0792cb5fb78cfac4e2ae196aaf9aa7d9c6 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 15:16:33 +0100
Subject: [PATCH 25/54] fix ruby script errors

---
 bin/compile                        |  4 +++-
 bin/supply                         |  1 +
 lib/scripts/update_agent_config.sh |  1 +
 lib/scripts/update_tags.rb         | 13 ++++++-------
 4 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/bin/compile b/bin/compile
index 19fdb81..825b1a9 100755
--- a/bin/compile
+++ b/bin/compile
@@ -68,4 +68,6 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
-python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
\ No newline at end of file
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
+
+date +%s > "${DATADOG_DIR}/startup_time"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index 857a0e5..d796e47 100755
--- a/bin/supply
+++ b/bin/supply
@@ -70,3 +70,4 @@ printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$
 # sanitize env vars and export a new a env file
 python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
 
+date +%s > "${DATADOG_DIR}/startup_time"
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index f8d3b37..da0bde9 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -66,6 +66,7 @@ main() {
     log_message "$0" "$$" "Writing DD_TAGS to node_agent_tags.txt"
     
     # update node_agent_tags.txt
+    log_message "$0" "$$" "Updating node_agent_tags.txt"
     ruby "${DATADOG_DIR}"/scripts/update_tags.rb
     
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index 4b4dc54..b9a3ee5 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -15,31 +15,30 @@
 node_agent_tags_file = File.join(DATADOG_DIR, "node_agent_tags.txt")
 
 # read startup time set by the buildpack supply script
-timestamp = File.read(timestamp_file).strip
-time = Time.parse(timestamp)
+timestamp = File.read(timestamp_file).strip.to_i
 
 # storing all tags on this variable
 tags = []
 
 if ! DD_NODE_AGENT_TAGS.empty?
     tags.concat(DD_NODE_AGENT_TAGS.split(','))
+end
 
 if ! DD_TAGS.empty?
     tags.concat(DD_TAGS.split(','))
+end
 
 # if the script is executed during the warmup period, merge incoming tags with the existing tags
 # otherwise, override existing tags
-if Time.now - time <= DD_UPDATE_SCRIPT_WARMUP.to_i
+if Time.now.to_i - timestamp <= DD_UPDATE_SCRIPT_WARMUP.to_i
     if File.exists?(node_agent_tags_file)
         node_tags = File.read(node_agent_tags_file).split(',')
         tags.concat(node_tags)
+    end
 end
 
 # remove duplicates
 tags = tags.uniq
 
 # export tags
-File.write(node_agent_tags_file, tags.join(","))
-
-
-
+File.write(node_agent_tags_file, tags.join(","))
\ No newline at end of file

From 4dad3d9b3aac9ecd5277dae32ec99298ab4ded36 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 22:33:00 +0100
Subject: [PATCH 26/54] filter tags with json values

---
 lib/scripts/update_tags.rb | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index b9a3ee5..ecccf23 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -20,12 +20,18 @@
 # storing all tags on this variable
 tags = []
 
+def sanitize(tags_env_var)
+    tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
+    tags_list.keep_if { |element| !element.include?(";") }
+    return tags_list
+end
+
 if ! DD_NODE_AGENT_TAGS.empty?
-    tags.concat(DD_NODE_AGENT_TAGS.split(','))
+    tags.concat(sanitize(DD_NODE_AGENT_TAGS))
 end
 
 if ! DD_TAGS.empty?
-    tags.concat(DD_TAGS.split(','))
+    tags.concat(sanitize(DD_TAGS))
 end
 
 # if the script is executed during the warmup period, merge incoming tags with the existing tags

From 4a7a6d7abc4a73ec19b576147dc4e468e380dc6c Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 22:33:15 +0100
Subject: [PATCH 27/54] add log levels

---
 lib/scripts/check_datadog.sh       | 10 ++++-----
 lib/scripts/update_agent_config.sh | 34 +++++++++++++++---------------
 lib/scripts/utils.sh               | 14 ++++++++++--
 3 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/lib/scripts/check_datadog.sh b/lib/scripts/check_datadog.sh
index e2fb4d6..a268854 100644
--- a/lib/scripts/check_datadog.sh
+++ b/lib/scripts/check_datadog.sh
@@ -10,19 +10,19 @@ source "${DATADOG_DIR}/scripts/utils.sh"
 
 check_datadog() {
   while true; do
-    log_message "$0" "$$" "Waiting for agent or dogstatsd process to start"
+    log_info "Waiting for agent or dogstatsd process to start"
     if check_if_running "${AGENT_PIDFILE}" "${AGENT_CMD}"; then
-      log_message "$0" "$$" "found agent process: $(cat "${AGENT_PIDFILE}")"
+      log_debug "found agent process: $(cat "${AGENT_PIDFILE}")"
       if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
-        log_message "$0" "$$" "found agent token"
+        log_info "found agent token"
         break
       else 
-        log_message "$0" "$$" "agent token not found"
+        log_info "agent token not found"
       fi
     fi
 
     if check_if_running "${DOGSTATSD_PIDFILE}" "${DOGSTATSD_CMD}"; then
-      log_message "$0" "$$" "found dogstatsd process: $(cat "${DOGSTATSD_PIDFILE}")"
+      log_info "found dogstatsd process: $(cat "${DOGSTATSD_PIDFILE}")"
       break
     fi
     sleep 1
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index da0bde9..212c105 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -11,30 +11,30 @@ LOCK="${DATADOG_DIR}/update.lock"
 . "${DATADOG_DIR}/scripts/utils.sh"
 
 release_lock() {
-    log_message "$0" "$$" "releasing lock '${LOCK}'"
+    log_info "releasing lock '${LOCK}'"
     rmdir "${LOCK}"
 }
 
 main() {
-    log_message "$0" "$$" "starting update_agent_config script"
-    log_message "$0" "$$" "(BEFORE)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
+    log_info "starting update_agent_config script"
+    log_debug "(BEFORE)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
 
     # try to create the LOCK
     while ! mkdir "${LOCK}" 2>/dev/null; do
-        log_message "$0" "$$" "cannot acquire lock, script is already running"
+        log_info "cannot acquire lock, script is already running"
         sleep 1
     done
 
-    log_message "$0" "$$" "acquired lock '${LOCK}'"
+    log_info "acquired lock '${LOCK}'"
    
     # ensures the lock is released on exit
     trap release_lock INT TERM EXIT
 
     # wait for the buildpack scripts to finish
-    log_message "$0" "$$" "starting check_datadog script"
+    log_info "starting check_datadog script"
 
     while ! [ -f "${DATADOG_DIR}/scripts/check_datadog.sh" ]; do
-        log_message "$0" "$$" "check_datadog.sh script not found, waiting..."
+        log_info "check_datadog.sh script not found, waiting..."
         sleep 2
     done
 
@@ -43,11 +43,11 @@ main() {
     
     # verify that check_datadog exited successfully 
     if [ ${exit_code} -ne  0 ]; then
-        log_message "$0" "$$" "could not find agent, aborting update script!"
+        log_error "could not find agent, aborting update script!"
         exit ${exit_code}
     fi
 
-    log_message "$0" "$$" "finished check_datadog script"
+    log_info "finished check_datadog script"
 
     # source relevant DD tags
     . "${DATADOG_DIR}/.datadog_env"
@@ -58,30 +58,30 @@ main() {
     export LOGS_CONFIG
 
     # update logs configs with the new tags
-    log_message "$0" "$$" "Creating logs config"
+    log_info "Creating logs config"
     ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
 
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
-    log_message "$0" "$$" "Writing DD_TAGS to node_agent_tags.txt"
+    log_info "Writing DD_TAGS to node_agent_tags.txt"
     
     # update node_agent_tags.txt
-    log_message "$0" "$$" "Updating node_agent_tags.txt"
+    log_info "Updating node_agent_tags.txt"
     ruby "${DATADOG_DIR}"/scripts/update_tags.rb
     
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
-    log_message "$0" "$$" "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
-    log_message "$0" "$$" "(AFTER)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
+    log_debug "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
+    log_debug "(AFTER)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
 
     # finishing up
-    log_message "$0" "$$" "exporting .sourced_datadog_env file"
+    log_info "exporting .sourced_datadog_env file"
     printenv > "${DATADOG_DIR}/.sourced_datadog_env"
 
     # mark to the monit_datadog function in run-datadog.sh that the script is finished
-    log_message "$0" "$$" "creating tags_updated file"
+    log_info "creating tags_updated file"
     touch "${DATADOG_DIR}/tags_updated"
 
-    log_message "$0" "$$" "finished update_agent_config script"
+    log_info "finished update_agent_config script"
 }
 
 # for debugging purposes
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 6e6a6a7..aadd8ae 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -21,13 +21,23 @@ export TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace
 export DOGSTATSD_PIDFILE="${DATADOG_DIR}/run/trace-agent.pid"
 export DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
 
+log_info() {
+  log_message "$0" "$$" "$@" "INFO"
+}
+
+log_debug() {
+  log_message "$0" "$$" "$@" "DEBUG"
+}
 
+log_error() {
+  log_message "$0" "$$" "$@" "ERROR" 1>&2
+}
 
 log_message() {
   local component="${1#/home/vcap/app/}"
   local pid="$2"
   local message="$3"
-  local log_level="${4:-INFO}"
+  local log_level="$4"
   echo "$(date +'%d-%m-%Y %H:%M:%S') - [${component}][PID:${pid}] - ${log_level} - ${message}"
 }
 
@@ -144,7 +154,7 @@ redirect() {
     else
         nc localhost "${STD_LOG_COLLECTION_PORT}" || sleep 0.5
     fi
-    log_message "$0" "$$" "Resetting buildpack log redirection"
+    log_info "Resetting buildpack log redirection"
     if [ "${DD_DEBUG_STD_REDIRECTION}" = "true" ]; then
       HTTP_PROXY=${DD_HTTP_PROXY} HTTPS_PROXY=${DD_HTTPS_PROXY} NO_PROXY=${DD_NO_PROXY} curl \
       -X POST -H "Content-type: application/json" \

From 807ac5eaa361b13e27a493f0b6334a6ccdf81241 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sat, 17 Dec 2022 22:45:52 +0100
Subject: [PATCH 28/54] move startup_time file creation into run-datadog.sh
 script to support cf restart scenarios

---
 bin/compile        | 4 +---
 bin/supply         | 2 --
 lib/run-datadog.sh | 1 +
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/bin/compile b/bin/compile
index 825b1a9..19fdb81 100755
--- a/bin/compile
+++ b/bin/compile
@@ -68,6 +68,4 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
-python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
-
-date +%s > "${DATADOG_DIR}/startup_time"
\ No newline at end of file
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index d796e47..40206fe 100755
--- a/bin/supply
+++ b/bin/supply
@@ -69,5 +69,3 @@ printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$
 
 # sanitize env vars and export a new a env file
 python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
-
-date +%s > "${DATADOG_DIR}/startup_time"
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 2ef1a9f..63bb44f 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -121,6 +121,7 @@ start_datadog() {
     export LOGS_CONFIG
 
     if [ "${FIRST_RUN}" = "true" ]; then
+      date +%s > "${DATADOG_DIR}/startup_time"
       echo "setting up datadog"
       setup_datadog
       FIRST_RUN=false

From d08595b27bfaaf82a2d5818dd5d8911fe2d7aba1 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sun, 18 Dec 2022 16:57:53 +0100
Subject: [PATCH 29/54] put capi metadata collection behind
 'DD_ENABLE_METADATA_COLLECTION' flag

---
 lib/run-datadog.sh                 |  1 +
 lib/scripts/update_agent_config.sh | 11 ++++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 63bb44f..4b7c6c7 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -6,6 +6,7 @@
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
+DD_ENABLE_METADATA_COLLECTION="${DD_ENABLE_METADATA_COLLECTION:-false}"
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 212c105..660b437 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -16,6 +16,14 @@ release_lock() {
 }
 
 main() {
+    # source relevant DD tags
+    . "${DATADOG_DIR}/.datadog_env"
+
+    if [ "${DD_ENABLE_METADATA_COLLECTION}" != "true" ]; then
+        log_info "update script aborted. set DD_ENABLE_METADATA_COLLECTION to true to enable metadata tags collection"
+        exit 0
+    fi
+
     log_info "starting update_agent_config script"
     log_debug "(BEFORE)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
 
@@ -49,9 +57,6 @@ main() {
 
     log_info "finished check_datadog script"
 
-    # source relevant DD tags
-    . "${DATADOG_DIR}/.datadog_env"
-
     # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
     export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)"
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"

From 66c3bf8811370cff22abb735417dd0e2df741fc4 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Sun, 18 Dec 2022 21:00:03 +0100
Subject: [PATCH 30/54] use a file as flag instead of sourcing datadog_env file
 to prevent tags from being overriden

---
 lib/run-datadog.sh                 | 3 +++
 lib/scripts/update_agent_config.sh | 8 ++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 4b7c6c7..f2e8465 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -223,6 +223,9 @@ monit_datadog() {
 }
 
 main() {
+  if [ "${DD_ENABLE_METADATA_COLLECTION}" != "true" ]; then
+        touch "${DATADOG_DIR}/.dd_enable_metadata_collection"
+  fi
   if [ -z "${DD_API_KEY}" ]; then
     echo "Datadog API Key not set, not starting Datadog"
   else
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 660b437..7b43ffa 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -16,10 +16,7 @@ release_lock() {
 }
 
 main() {
-    # source relevant DD tags
-    . "${DATADOG_DIR}/.datadog_env"
-
-    if [ "${DD_ENABLE_METADATA_COLLECTION}" != "true" ]; then
+    if [ -f "${DATADOG_DIR}/.dd_enable_metadata_collection" ]; then
         log_info "update script aborted. set DD_ENABLE_METADATA_COLLECTION to true to enable metadata tags collection"
         exit 0
     fi
@@ -57,6 +54,9 @@ main() {
 
     log_info "finished check_datadog script"
 
+    # source relevant DD tags
+    . "${DATADOG_DIR}/.datadog_env"
+
     # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
     export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)"
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"

From 5a72ba0194cfc4dc0a3e6d668f4b0dc09dafbfe6 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 12:24:48 +0100
Subject: [PATCH 31/54] remove common.sh

---
 bin/compile           |  2 --
 bin/supply            |  2 --
 lib/scripts/common.sh | 20 --------------------
 3 files changed, 24 deletions(-)
 delete mode 100644 lib/scripts/common.sh

diff --git a/bin/compile b/bin/compile
index 19fdb81..07f4b55 100755
--- a/bin/compile
+++ b/bin/compile
@@ -38,7 +38,6 @@ cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.
 
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
-cp "${ROOT_DIR}/lib/scripts/common.sh" "${DATADOG_DIR}/scripts/common.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
@@ -54,7 +53,6 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
 fi
 
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
-chmod +x "${DATADOG_DIR}/scripts/common.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
diff --git a/bin/supply b/bin/supply
index 40206fe..5ded2c4 100755
--- a/bin/supply
+++ b/bin/supply
@@ -38,7 +38,6 @@ cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
-cp "${ROOT_DIR}/lib/scripts/common.sh" "${DATADOG_DIR}/scripts/common.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
@@ -54,7 +53,6 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
 fi
 
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
-chmod +x "${DATADOG_DIR}/scripts/common.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
 chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
diff --git a/lib/scripts/common.sh b/lib/scripts/common.sh
deleted file mode 100644
index 010c7bc..0000000
--- a/lib/scripts/common.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
-# This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2022-Present Datadog, Inc.
-
-export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
-export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
-
-export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
-export LOGS_CONFIG
-
-
-log_message() {
-  local component="${1#/home/vcap/app/}"
-  local pid="$2"
-  local message="$3"
-  local log_level="${4:-INFO}"
-  echo "$(date +'%d-%m-%Y %H:%M:%S') - [${component}][PID:${pid}] - ${log_level} - ${message}"
-}
\ No newline at end of file

From b820a5bfb10bbaa26ddff2460e4781397f613fa4 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 12:41:31 +0100
Subject: [PATCH 32/54] remove unused script

---
 bin/compile                       |  1 -
 bin/supply                        |  1 -
 lib/scripts/update_yaml_config.rb | 24 ------------------------
 3 files changed, 26 deletions(-)
 delete mode 100644 lib/scripts/update_yaml_config.rb

diff --git a/bin/compile b/bin/compile
index 07f4b55..a4b2a92 100755
--- a/bin/compile
+++ b/bin/compile
@@ -33,7 +33,6 @@ cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
-cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
diff --git a/bin/supply b/bin/supply
index 5ded2c4..9d27a76 100755
--- a/bin/supply
+++ b/bin/supply
@@ -32,7 +32,6 @@ cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
-cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
diff --git a/lib/scripts/update_yaml_config.rb b/lib/scripts/update_yaml_config.rb
deleted file mode 100644
index 2079732..0000000
--- a/lib/scripts/update_yaml_config.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
-# This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2022-Present Datadog, Inc.
-
-#!/usr/bin/env ruby
-
-require 'yaml'
-
-# env vars
-DATADOG_DIR = ENV.fetch('DATADOG_DIR', '/home/vcap/app/.datadog')
-DD_TAGS = ENV.fetch('DD_TAGS', '')
-DD_NODE_AGENT_TAGS = ENV.fetch('DD_NODE_AGENT_TAGS', '')
-
-
-datadog_config_filepath = File.join(DATADOG_DIR, 'dist/datadog.yaml')
-
-
-file = File.open(datadog_config_filepath, 'r')
-yaml_data = YAML.load(file)
-
-tags = DD_TAGS.split(',') + DD_NODE_AGENT_TAGS.split(',')
-yaml_data['tags'] = tags.uniq
-
-File.write(datadog_config_filepath, yaml_data.to_yaml)

From 24a05a53a99f786345ee20458c0bfa5c16c7873c Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 12:54:14 +0100
Subject: [PATCH 33/54] remove cp commands to deleted script

---
 bin/compile | 2 --
 bin/supply  | 2 --
 2 files changed, 4 deletions(-)

diff --git a/bin/compile b/bin/compile
index a4b2a92..5174c0c 100755
--- a/bin/compile
+++ b/bin/compile
@@ -39,7 +39,6 @@ cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
 cp "${ROOT_DIR}/lib/run-datadog.sh" "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 cp "${ROOT_DIR}/lib/redirect-logs.sh" "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
@@ -54,7 +53,6 @@ fi
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 chmod +x "${DATADOG_DIR}/trace-agent"
 chmod +x "${BUILD_DIR}/.profile.d/00-test-endpoint.sh"
 chmod +x "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
diff --git a/bin/supply b/bin/supply
index 9d27a76..3be3958 100755
--- a/bin/supply
+++ b/bin/supply
@@ -39,7 +39,6 @@ cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
 cp "${ROOT_DIR}/lib/run-datadog.sh" "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 cp "${ROOT_DIR}/lib/redirect-logs.sh" "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
@@ -54,7 +53,6 @@ fi
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 chmod +x "${DATADOG_DIR}/trace-agent"
 chmod +x "${BUILD_DIR}/.profile.d/00-test-endpoint.sh"
 chmod +x "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"

From 1c55c1eebb60051f466efb7ff438c178048b52b1 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 13:53:31 +0100
Subject: [PATCH 34/54] nits

---
 lib/run-datadog.sh                | 2 +-
 lib/scripts/create_logs_config.rb | 8 --------
 lib/scripts/utils.sh              | 2 +-
 3 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index f2e8465..7d38655 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -10,7 +10,7 @@ DD_ENABLE_METADATA_COLLECTION="${DD_ENABLE_METADATA_COLLECTION:-false}"
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 
-source ${DATADOG_DIR}/scripts/utils.sh
+source "${DATADOG_DIR}/scripts/utils.sh"
 
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index 16cf730..c1cddf4 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -9,14 +9,6 @@
 dd_env_file = "/home/vcap/app/.datadog/.sourced_env_datadog"
 node_agent_tags = "/home/vcap/app/.datadog/node_agent_tags.txt"
 
-if File.file?(dd_env_file)
-  File.readlines().each do |line|
-    values = line.split("=")
-    ENV[values[0]] = values[1]
-  end
-  puts "ruby: sourced datadog env file"
-end
-
 logs_config_dir = ENV['LOGS_CONFIG_DIR']
 logs_config = ENV['LOGS_CONFIG']
 dd_tags = ENV['DD_TAGS']
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index aadd8ae..4511364 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -58,7 +58,7 @@ wait_pid() {
   local try_kill="$3"
   local timeout="${4:-0}"
   local force="${5:-0}"
-  local countdown=$(( 100 )) # temporary to workaround a /bin/dash syntax error
+  local countdown=$(( ${timeout} * 10 ))
   local ps_out="$(ps ax | grep ${pid} | grep -v grep)"
 
   if [ -e "/proc/${pid}" ] || [ -n "${ps_out}" ]; then

From 5ef237c4fce5f13e57b746d0460ea4f75e44af09 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 14:15:59 +0100
Subject: [PATCH 35/54] Revert "remove unused script"

This reverts commit b820a5bfb10bbaa26ddff2460e4781397f613fa4.
---
 bin/compile                       |  1 +
 bin/supply                        |  1 +
 lib/scripts/update_yaml_config.rb | 24 ++++++++++++++++++++++++
 3 files changed, 26 insertions(+)
 create mode 100644 lib/scripts/update_yaml_config.rb

diff --git a/bin/compile b/bin/compile
index 5174c0c..a619a55 100755
--- a/bin/compile
+++ b/bin/compile
@@ -33,6 +33,7 @@ cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
+cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
diff --git a/bin/supply b/bin/supply
index 3be3958..8ab95e3 100755
--- a/bin/supply
+++ b/bin/supply
@@ -32,6 +32,7 @@ cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
diff --git a/lib/scripts/update_yaml_config.rb b/lib/scripts/update_yaml_config.rb
new file mode 100644
index 0000000..2079732
--- /dev/null
+++ b/lib/scripts/update_yaml_config.rb
@@ -0,0 +1,24 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022-Present Datadog, Inc.
+
+#!/usr/bin/env ruby
+
+require 'yaml'
+
+# env vars
+DATADOG_DIR = ENV.fetch('DATADOG_DIR', '/home/vcap/app/.datadog')
+DD_TAGS = ENV.fetch('DD_TAGS', '')
+DD_NODE_AGENT_TAGS = ENV.fetch('DD_NODE_AGENT_TAGS', '')
+
+
+datadog_config_filepath = File.join(DATADOG_DIR, 'dist/datadog.yaml')
+
+
+file = File.open(datadog_config_filepath, 'r')
+yaml_data = YAML.load(file)
+
+tags = DD_TAGS.split(',') + DD_NODE_AGENT_TAGS.split(',')
+yaml_data['tags'] = tags.uniq
+
+File.write(datadog_config_filepath, yaml_data.to_yaml)

From 6a21ce66d0afd92a74876ab956af0f3386d7ff4b Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 18:01:24 +0100
Subject: [PATCH 36/54] fix DD_TAGS format issues

---
 lib/run-datadog.sh                |  5 +++--
 lib/scripts/update_yaml_config.rb | 11 +++++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 7d38655..c05fa2b 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -12,8 +12,6 @@ FIRST_RUN="${FIRST_RUN:-true}"
 
 source "${DATADOG_DIR}/scripts/utils.sh"
 
-export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-
 setup_datadog() {
   pushd "${DATADOG_DIR}"
 
@@ -214,6 +212,9 @@ monit_datadog() {
       echo "tags_updated found, stopping datadog agents"
       stop_datadog
       echo "tags_updated found, starting datadog agents"
+      export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+      ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+      unset DD_TAGS
       start_datadog
       echo "deleting tags_updated"
       rm -f "${DATADOG_DIR}"/tags_updated # TODO: check for race conditions
diff --git a/lib/scripts/update_yaml_config.rb b/lib/scripts/update_yaml_config.rb
index 2079732..fcbf613 100644
--- a/lib/scripts/update_yaml_config.rb
+++ b/lib/scripts/update_yaml_config.rb
@@ -11,14 +11,21 @@
 DD_TAGS = ENV.fetch('DD_TAGS', '')
 DD_NODE_AGENT_TAGS = ENV.fetch('DD_NODE_AGENT_TAGS', '')
 
+def sanitize(tags_env_var)
+    tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
+    tags_list.keep_if { |element| !element.include?(";") }
+    return tags_list
+end
 
-datadog_config_filepath = File.join(DATADOG_DIR, 'dist/datadog.yaml')
 
+datadog_config_filepath = File.join(DATADOG_DIR, 'dist/datadog.yaml')
 
 file = File.open(datadog_config_filepath, 'r')
 yaml_data = YAML.load(file)
 
-tags = DD_TAGS.split(',') + DD_NODE_AGENT_TAGS.split(',')
+tags = sanitize(DD_TAGS) + sanitize(DD_NODE_AGENT_TAGS)
+
+# remove duplicates
 yaml_data['tags'] = tags.uniq
 
 File.write(datadog_config_filepath, yaml_data.to_yaml)

From b10585c38a6700c5a20812958c5de8fbb4d79050 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Mon, 19 Dec 2022 18:04:55 +0100
Subject: [PATCH 37/54] fix dogstatsd pidfile name as suggested

---
 lib/scripts/utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index 4511364..c08bbd0 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -18,7 +18,7 @@ export AGENT_CMD="./agent run --cfgpath dist/ --pidfile run/agent.pid"
 export TRACE_AGENT_PIDFILE="${DATADOG_DIR}/run/trace-agent.pid"
 export TRACE_AGENT_CMD="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
 
-export DOGSTATSD_PIDFILE="${DATADOG_DIR}/run/trace-agent.pid"
+export DOGSTATSD_PIDFILE="${DATADOG_DIR}/run/dogstatsd.pid"
 export DOGSTATSD_CMD="./dogstatsd start --cfgpath dist/"
 
 log_info() {

From c0131b8e3c3e18abb5a825213040bde7c199a9db Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 21 Dec 2022 19:13:20 +0100
Subject: [PATCH 38/54] apply suggestions

---
 lib/scripts/update_agent_config.sh | 9 ++++++---
 lib/scripts/update_tags.rb         | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 7b43ffa..f722204 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -63,9 +63,12 @@ main() {
     export LOGS_CONFIG
 
     # update logs configs with the new tags
-    log_info "Creating logs config"
-    ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
-
+    if [ -n "${LOGS_CONFIG}" ]; then
+        mkdir -p "${LOGS_CONFIG_DIR}"
+        log_info "Creating logs config"
+        ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
+    fi
+    
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
     log_info "Writing DD_TAGS to node_agent_tags.txt"
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index ecccf23..f1bfe60 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -15,7 +15,7 @@
 node_agent_tags_file = File.join(DATADOG_DIR, "node_agent_tags.txt")
 
 # read startup time set by the buildpack supply script
-timestamp = File.read(timestamp_file).strip.to_i
+timestamp = File.exists?(timestamp_file) ? File.read(timestamp_file).strip.to_i : 0
 
 # storing all tags on this variable
 tags = []

From e5d2e3131e25772603b5f8bfecab515f6927367a Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 21 Dec 2022 19:22:45 +0100
Subject: [PATCH 39/54] sanitize tags everywhere + add support for space
 separated user-tags

---
 lib/scripts/create_logs_config.rb | 10 ++++++++--
 lib/scripts/get_tags.py           | 17 +++++++++++++----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index c1cddf4..410f6ca 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -14,6 +14,12 @@
 dd_tags = ENV['DD_TAGS']
 dd_node_agent_tags = ENV['DD_NODE_AGENTS_TAGS'] || (File.file?(node_agent_tags) ? File.read(node_agent_tags) : "")
 
+def sanitize(tags_env_var)
+  tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
+  tags_list.keep_if { |element| !element.include?(";") }
+  return tags_list
+end
+
 config = {}
 
 if logs_config_dir.nil?
@@ -27,14 +33,14 @@
   tags_list = []
 
   if !dd_tags.nil?
-    tags_list += dd_tags.split(',')
+    tags_list += sanitize(dd_tags)
     # puts "DD_TAGS found in ruby script=#{dd_tags}"
   else
     puts "Could not find DD_TAGS env var"
   end
 
   if !dd_node_agent_tags.nil?
-    tags_list += dd_node_agent_tags.split(',')
+    tags_list += sanitize(dd_node_agent_tags)
     # puts "DD_NODE_AGENTS_TAGS found in ruby script=#{dd_node_agent_tags}"
   else
     puts "Could not find DD_NODE_AGENTS_TAGS env var"
diff --git a/lib/scripts/get_tags.py b/lib/scripts/get_tags.py
index 81f0d75..6c1a699 100644
--- a/lib/scripts/get_tags.py
+++ b/lib/scripts/get_tags.py
@@ -8,6 +8,14 @@
 import json
 import sys
 
+def parse_tags(tags):
+    delimiter = ','
+    if ' ' in tags:
+        delimiter = ' '
+    try:
+        return tags.split(delimiter)
+    except Exception as e:
+        print("there was an issue parsing the tags in {tags.__name__}: {}".format(e))
 
 vcap_app_string = os.environ.get('VCAP_APPLICATION', '{}')
 
@@ -30,7 +38,7 @@
     if node_agent_tags is not None:
         # we do this to separate commas inside json values from tags separator commas
         node_agent_tags = node_agent_tags.replace(",\"", ";\"")
-        all_node_agent_tags = node_agent_tags.split(",")
+        all_node_agent_tags = parse_tags(node_agent_tags)
         tags = tags + [tag for tag in all_node_agent_tags if ";" not in tag]
 
 
@@ -50,7 +58,7 @@
 user_tags = os.environ.get('TAGS', None)
 if user_tags:
     try:
-        user_tags = user_tags.split(',')
+        user_tags = parse_tags(user_tags)
         for tag in user_tags:
             tags.append(tag)
     except Exception as e:
@@ -59,7 +67,7 @@
 user_tags = os.environ.get('DD_TAGS', None)
 if user_tags:
     try:
-        user_tags = user_tags.split(',')
+        user_tags = parse_tags(user_tags)
         for tag in user_tags:
             tags.append(tag)
     except Exception as e:
@@ -67,8 +75,9 @@
 
 tags = [ tag.replace(" ", "_") for tag in tags ]
 tags = list(dict.fromkeys(tags))
+
 legacy_tags = os.environ.get('LEGACY_TAGS_FORMAT', False)
 if legacy_tags:
     print(','.join(tags))
 else:
-    print(json.dumps(tags))
+    print(json.dumps(tags))
\ No newline at end of file

From 0cb2ecc4d038e09fb5628d6f90b7b3e0c93ebfd7 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Tue, 27 Dec 2022 15:25:21 +0100
Subject: [PATCH 40/54] fix single tags, add support for space separated tags

---
 lib/run-datadog.sh            | 4 ++--
 lib/scripts/get_tags.py       | 2 +-
 lib/scripts/parse_env_vars.py | 9 +++++++--
 lib/scripts/update_tags.rb    | 1 +
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index c05fa2b..3b2e788 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -110,6 +110,8 @@ setup_datadog() {
 }
 
 start_datadog() {
+  export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+
   pushd "${DATADOG_DIR}"
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
     export DD_API_KEY
@@ -212,9 +214,7 @@ monit_datadog() {
       echo "tags_updated found, stopping datadog agents"
       stop_datadog
       echo "tags_updated found, starting datadog agents"
-      export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
       ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
-      unset DD_TAGS
       start_datadog
       echo "deleting tags_updated"
       rm -f "${DATADOG_DIR}"/tags_updated # TODO: check for race conditions
diff --git a/lib/scripts/get_tags.py b/lib/scripts/get_tags.py
index 6c1a699..185efb6 100644
--- a/lib/scripts/get_tags.py
+++ b/lib/scripts/get_tags.py
@@ -10,7 +10,7 @@
 
 def parse_tags(tags):
     delimiter = ','
-    if ' ' in tags:
+    if tags.count(' ') > tags.count(','):
         delimiter = ' '
     try:
         return tags.split(delimiter)
diff --git a/lib/scripts/parse_env_vars.py b/lib/scripts/parse_env_vars.py
index fb2a7b7..972fda3 100644
--- a/lib/scripts/parse_env_vars.py
+++ b/lib/scripts/parse_env_vars.py
@@ -16,8 +16,13 @@
                 env_var = env_var.strip()
                 if env_var.startswith("#"):
                     continue
-                env_var = env_var.replace(" ", "_")
-                new_env_file.write("export {}\n".format(env_var))
+                env_parts = env_var.split("=", 1)
+                if len(env_parts) != 2:
+                    continue
+                name, value = env_parts
+                if name not in ["DD_TAGS", "TAGS"]:
+                    value = value.replace(" ", "_")
+                new_env_file.write("export {}={}\n".format(name, value))
     else:
         print("Destination file not specified")
 else:
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index f1bfe60..726164a 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -23,6 +23,7 @@
 def sanitize(tags_env_var)
     tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
     tags_list.keep_if { |element| !element.include?(";") }
+    tags_list.each { |tag| tag = tag.gsub(" ", "_") unless !tag.include? "org" }
     return tags_list
 end
 

From 82276a9362169ca8c3a1614642fd3843cd44a9dc Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Tue, 27 Dec 2022 18:01:58 +0100
Subject: [PATCH 41/54] WIP: cleanup scripts

---
 lib/run-datadog.sh                 |  6 ++++--
 lib/scripts/create_logs_config.rb  |  2 --
 lib/scripts/get_tags.py            | 22 ++++++++++++----------
 lib/scripts/update_agent_config.sh |  6 ++----
 lib/scripts/update_tags.rb         |  2 +-
 lib/scripts/update_yaml_config.rb  |  1 +
 6 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 3b2e788..e7dbae5 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -9,6 +9,7 @@ SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 DD_ENABLE_METADATA_COLLECTION="${DD_ENABLE_METADATA_COLLECTION:-false}"
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
+USER_TAGS="${DD_TAGS}"
 
 source "${DATADOG_DIR}/scripts/utils.sh"
 
@@ -36,7 +37,6 @@ setup_datadog() {
     # add logs configs
     if [ -n "${LOGS_CONFIG}" ]; then
       mkdir -p ${LOGS_CONFIG_DIR}
-      ruby scripts/create_logs_config.rb
     fi
 
     # The yaml file requires the tags to be an array,
@@ -110,6 +110,7 @@ setup_datadog() {
 }
 
 start_datadog() {
+  DD_TAGS="${USER_TAGS}"
   export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 
   pushd "${DATADOG_DIR}"
@@ -144,8 +145,10 @@ start_datadog() {
         export DD_IOT_HOST=false
 
         # update logs configs
+        # TODO: move it to setup datadog and let the update script take care of updating the logs config
         if [ -n "${LOGS_CONFIG}" ]; then
           mkdir -p "${LOGS_CONFIG_DIR}"
+          echo "creating logs config from start_datadog"
           ruby scripts/create_logs_config.rb
         fi
 
@@ -214,7 +217,6 @@ monit_datadog() {
       echo "tags_updated found, stopping datadog agents"
       stop_datadog
       echo "tags_updated found, starting datadog agents"
-      ruby "${DATADOG_DIR}/scripts/update_yaml_config.rb"
       start_datadog
       echo "deleting tags_updated"
       rm -f "${DATADOG_DIR}"/tags_updated # TODO: check for race conditions
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index 410f6ca..225a940 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -34,14 +34,12 @@ def sanitize(tags_env_var)
 
   if !dd_tags.nil?
     tags_list += sanitize(dd_tags)
-    # puts "DD_TAGS found in ruby script=#{dd_tags}"
   else
     puts "Could not find DD_TAGS env var"
   end
 
   if !dd_node_agent_tags.nil?
     tags_list += sanitize(dd_node_agent_tags)
-    # puts "DD_NODE_AGENTS_TAGS found in ruby script=#{dd_node_agent_tags}"
   else
     puts "Could not find DD_NODE_AGENTS_TAGS env var"
   end
diff --git a/lib/scripts/get_tags.py b/lib/scripts/get_tags.py
index 185efb6..8304b92 100644
--- a/lib/scripts/get_tags.py
+++ b/lib/scripts/get_tags.py
@@ -8,6 +8,9 @@
 import json
 import sys
 
+# if DD_TAGS[0] is comma or space, then set is as delimiter
+# else continue as usual
+
 def parse_tags(tags):
     delimiter = ','
     if tags.count(' ') > tags.count(','):
@@ -31,16 +34,15 @@ def parse_tags(tags):
 
 tags.append("container_id:{}".format(os.environ.get("CF_INSTANCE_GUID")))
 
-if len(sys.argv) > 1 and sys.argv[1] == 'node-agent-tags':
-    # These are always comma separated - See https://github.com/DataDog/datadog-agent/blob/main/pkg/cloudfoundry/containertagger/container_tagger.go#L133
-    node_agent_tags = os.environ.get('DD_NODE_AGENT_TAGS', None)
-
-    if node_agent_tags is not None:
-        # we do this to separate commas inside json values from tags separator commas
-        node_agent_tags = node_agent_tags.replace(",\"", ";\"")
-        all_node_agent_tags = parse_tags(node_agent_tags)
-        tags = tags + [tag for tag in all_node_agent_tags if ";" not in tag]
-
+node_agent_tags = os.environ.get('DD_NODE_AGENT_TAGS', None)
+if node_agent_tags:
+    # These are always comma separated
+    # See https://github.com/DataDog/datadog-agent/blob/main/pkg/cloudfoundry/containertagger/container_tagger.go#L133
+    
+    # we do this to separate commas inside json values from tags separator commas
+    node_agent_tags = node_agent_tags.replace(",\"", ";\"")
+    all_node_agent_tags = parse_tags(node_agent_tags)
+    tags = tags + [tag for tag in all_node_agent_tags if ";" not in tag]
 
 for vcap_var_name in vcap_variables:
     vcap_var = vcap_application.get(vcap_var_name)
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index f722204..eede052 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -58,7 +58,7 @@ main() {
     . "${DATADOG_DIR}/.datadog_env"
 
     # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
-    export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py" node-agent-tags)"
+    export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py")"
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
 
@@ -71,11 +71,9 @@ main() {
     
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
-    log_info "Writing DD_TAGS to node_agent_tags.txt"
-    
     # update node_agent_tags.txt
     log_info "Updating node_agent_tags.txt"
-    ruby "${DATADOG_DIR}"/scripts/update_tags.rb
+    ruby "${DATADOG_DIR}"/scripts/update_tags.rb"
     
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
     log_debug "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index 726164a..d8dd8ab 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -23,7 +23,7 @@
 def sanitize(tags_env_var)
     tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
     tags_list.keep_if { |element| !element.include?(";") }
-    tags_list.each { |tag| tag = tag.gsub(" ", "_") unless !tag.include? "org" }
+    tags_list.each { |tag| tag = tag.gsub(" ", "_") unless !tag.include? "org_name" }
     return tags_list
 end
 
diff --git a/lib/scripts/update_yaml_config.rb b/lib/scripts/update_yaml_config.rb
index fcbf613..bd729f8 100644
--- a/lib/scripts/update_yaml_config.rb
+++ b/lib/scripts/update_yaml_config.rb
@@ -26,6 +26,7 @@ def sanitize(tags_env_var)
 tags = sanitize(DD_TAGS) + sanitize(DD_NODE_AGENT_TAGS)
 
 # remove duplicates
+# TODO: add dogstatsd_tags
 yaml_data['tags'] = tags.uniq
 
 File.write(datadog_config_filepath, yaml_data.to_yaml)

From 8f0d7a716b5c4136a76bd698bb32772a9a577852 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Tue, 27 Dec 2022 15:59:56 -0500
Subject: [PATCH 42/54] Fix syntax error

---
 lib/scripts/update_agent_config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index eede052..7e0b1b7 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -73,7 +73,7 @@ main() {
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
     # update node_agent_tags.txt
     log_info "Updating node_agent_tags.txt"
-    ruby "${DATADOG_DIR}"/scripts/update_tags.rb"
+    ruby "${DATADOG_DIR}/scripts/update_tags.rb"
     
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
     log_debug "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"

From d94451e1ec5c6362160e1fffd9768dd713ff2efa Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 28 Dec 2022 17:48:48 +0100
Subject: [PATCH 43/54] add dogstatsd tags, fix race condition with
 .dd_enable_metadata_collection file, cleanup code, remove ruby scripts

---
 lib/run-datadog.sh                 | 16 ++++-----------
 lib/scripts/update_agent_config.sh | 14 +++++++------
 lib/scripts/update_tags.rb         |  4 ++--
 lib/scripts/update_yaml_config.rb  | 32 ------------------------------
 4 files changed, 14 insertions(+), 52 deletions(-)
 delete mode 100644 lib/scripts/update_yaml_config.rb

diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index e7dbae5..35d2337 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -6,7 +6,7 @@
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
-DD_ENABLE_METADATA_COLLECTION="${DD_ENABLE_METADATA_COLLECTION:-false}"
+DD_ENABLE_CAPI_METADATA_COLLECTION="${DD_ENABLE_CAPI_METADATA_COLLECTION:-false}"
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 USER_TAGS="${DD_TAGS}"
@@ -37,6 +37,8 @@ setup_datadog() {
     # add logs configs
     if [ -n "${LOGS_CONFIG}" ]; then
       mkdir -p ${LOGS_CONFIG_DIR}
+      echo "creating logs config"
+      ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
     fi
 
     # The yaml file requires the tags to be an array,
@@ -112,6 +114,7 @@ setup_datadog() {
 start_datadog() {
   DD_TAGS="${USER_TAGS}"
   export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+  export DD_DOGSTATSD_TAGS="${DD_TAGS}"
 
   pushd "${DATADOG_DIR}"
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
@@ -144,14 +147,6 @@ start_datadog() {
         export DD_LOG_FILE=agent.log
         export DD_IOT_HOST=false
 
-        # update logs configs
-        # TODO: move it to setup datadog and let the update script take care of updating the logs config
-        if [ -n "${LOGS_CONFIG}" ]; then
-          mkdir -p "${LOGS_CONFIG_DIR}"
-          echo "creating logs config from start_datadog"
-          ruby scripts/create_logs_config.rb
-        fi
-
         echo "Starting Datadog agent"
         if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
           ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
@@ -226,9 +221,6 @@ monit_datadog() {
 }
 
 main() {
-  if [ "${DD_ENABLE_METADATA_COLLECTION}" != "true" ]; then
-        touch "${DATADOG_DIR}/.dd_enable_metadata_collection"
-  fi
   if [ -z "${DD_API_KEY}" ]; then
     echo "Datadog API Key not set, not starting Datadog"
   else
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 7e0b1b7..f900fcf 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -16,8 +16,11 @@ release_lock() {
 }
 
 main() {
-    if [ -f "${DATADOG_DIR}/.dd_enable_metadata_collection" ]; then
-        log_info "update script aborted. set DD_ENABLE_METADATA_COLLECTION to true to enable metadata tags collection"
+    # source relevant DD tags
+    . "${DATADOG_DIR}/.datadog_env"
+
+    if [ "${DD_ENABLE_CAPI_METADATA_COLLECTION}" != "true" ]; then
+        log_info "update script aborted. set DD_ENABLE_CAPI_METADATA_COLLECTION to true to enable metadata tags collection"
         exit 0
     fi
 
@@ -54,8 +57,7 @@ main() {
 
     log_info "finished check_datadog script"
 
-    # source relevant DD tags
-    . "${DATADOG_DIR}/.datadog_env"
+   
 
     # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
     export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py")"
@@ -65,10 +67,10 @@ main() {
     # update logs configs with the new tags
     if [ -n "${LOGS_CONFIG}" ]; then
         mkdir -p "${LOGS_CONFIG_DIR}"
-        log_info "Creating logs config"
+        log_info "Updating logs config"
         ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
     fi
-    
+
     # the agent cloud_foundry_container workloadmeta collector reads from this file
     # See: https://github.com/DataDog/datadog-agent/blob/main/pkg/workloadmeta/collectors/internal/cloudfoundry/cf_container/cloudfoundry_container.go#L24
     # update node_agent_tags.txt
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index d8dd8ab..c8c1560 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -23,8 +23,8 @@
 def sanitize(tags_env_var)
     tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
     tags_list.keep_if { |element| !element.include?(";") }
-    tags_list.each { |tag| tag = tag.gsub(" ", "_") unless !tag.include? "org_name" }
-    return tags_list
+    tags_list.each { |tag| tag = tag.gsub(" ", "_") }
+    return tags_list.uniq
 end
 
 if ! DD_NODE_AGENT_TAGS.empty?
diff --git a/lib/scripts/update_yaml_config.rb b/lib/scripts/update_yaml_config.rb
deleted file mode 100644
index bd729f8..0000000
--- a/lib/scripts/update_yaml_config.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
-# This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2022-Present Datadog, Inc.
-
-#!/usr/bin/env ruby
-
-require 'yaml'
-
-# env vars
-DATADOG_DIR = ENV.fetch('DATADOG_DIR', '/home/vcap/app/.datadog')
-DD_TAGS = ENV.fetch('DD_TAGS', '')
-DD_NODE_AGENT_TAGS = ENV.fetch('DD_NODE_AGENT_TAGS', '')
-
-def sanitize(tags_env_var)
-    tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
-    tags_list.keep_if { |element| !element.include?(";") }
-    return tags_list
-end
-
-
-datadog_config_filepath = File.join(DATADOG_DIR, 'dist/datadog.yaml')
-
-file = File.open(datadog_config_filepath, 'r')
-yaml_data = YAML.load(file)
-
-tags = sanitize(DD_TAGS) + sanitize(DD_NODE_AGENT_TAGS)
-
-# remove duplicates
-# TODO: add dogstatsd_tags
-yaml_data['tags'] = tags.uniq
-
-File.write(datadog_config_filepath, yaml_data.to_yaml)

From aeda38d90ba0bca1f11637f080bd69fbd1a8f2af Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 28 Dec 2022 20:15:13 +0100
Subject: [PATCH 44/54] fix source command errors by correctly exporting
 .source_datadog_env file

---
 lib/scripts/update_agent_config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index f900fcf..bfd092d 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -83,7 +83,7 @@ main() {
 
     # finishing up
     log_info "exporting .sourced_datadog_env file"
-    printenv > "${DATADOG_DIR}/.sourced_datadog_env"
+    export > "${DATADOG_DIR}/.sourced_datadog_env"
 
     # mark to the monit_datadog function in run-datadog.sh that the script is finished
     log_info "creating tags_updated file"

From dcc47fe2520d0e16809a390fcc1ec6b9438aafe1 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 28 Dec 2022 20:18:11 +0100
Subject: [PATCH 45/54] remove unused script

---
 bin/compile                       |  2 --
 bin/supply                        |  2 --
 lib/scripts/create_logs_config.py | 43 -------------------------------
 3 files changed, 47 deletions(-)
 delete mode 100644 lib/scripts/create_logs_config.py

diff --git a/bin/compile b/bin/compile
index a619a55..36ea0b2 100755
--- a/bin/compile
+++ b/bin/compile
@@ -30,10 +30,8 @@ if [ -f "${ROOT_DIR}/lib/dogstatsd" ]; then
 fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
-cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
-cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
diff --git a/bin/supply b/bin/supply
index 8ab95e3..6683604 100755
--- a/bin/supply
+++ b/bin/supply
@@ -30,9 +30,7 @@ if [ -f "${ROOT_DIR}/lib/dogstatsd" ]; then
 fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
-cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
-cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
diff --git a/lib/scripts/create_logs_config.py b/lib/scripts/create_logs_config.py
deleted file mode 100644
index 71dc479..0000000
--- a/lib/scripts/create_logs_config.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
-# This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2017-Present Datadog, Inc.
-
-from __future__ import print_function
-
-import os
-import json
-
-LOGS_CONFIG_DIR = os.environ.get('LOGS_CONFIG_DIR')
-LOGS_CONFIG = os.environ.get('LOGS_CONFIG')
-DD_TAGS = os.environ.get('DD_TAGS')
-
-config = {}
-
-if LOGS_CONFIG_DIR is None:
-  print("ERROR: `LOGS_CONFIG_DIR` must be set in order to collect logs. For more info, see: https://github.com/DataDog/datadog-cloudfoundry-buildpack#log-collection")
-  exit(1)
-
-if LOGS_CONFIG is not None:
-  config["logs"] = json.loads(LOGS_CONFIG)
-
-  if DD_TAGS is not None:
-    config["logs"][0]["tags"] = DD_TAGS
-  else:
-    print("Could not find DD_TAGS env var")
-
-else:
-  print("ERROR: `LOGS_CONFIG` must be set in order to collect logs. For more info, see: https://github.com/DataDog/datadog-cloudfoundry-buildpack#log-collection")
-  exit(1)
-
-config = json.dumps(config)
-
-path = LOGS_CONFIG_DIR + "/logs.yaml"
-try:
-  if not os.path.exists(LOGS_CONFIG_DIR):
-    os.makedirs(LOGS_CONFIG_DIR)
-  with open(path, 'w+') as f:
-    print("writing {} to {}".format(config, path))
-    f.write(config)
-    f.write("\n")
-except Exception as e:
-  print("Could not write to log file: {}".format(str(e)))
\ No newline at end of file

From 972d7088f42d13e483025d81c5b6dc05098a86b1 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Wed, 28 Dec 2022 15:49:27 -0500
Subject: [PATCH 46/54] Parse out spaces

---
 lib/scripts/update_tags.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index c8c1560..4f3b699 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -23,7 +23,7 @@
 def sanitize(tags_env_var)
     tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
     tags_list.keep_if { |element| !element.include?(";") }
-    tags_list.each { |tag| tag = tag.gsub(" ", "_") }
+    tags_list = tags_list.map { |tag| tag.gsub(" ", "_") }
     return tags_list.uniq
 end
 

From 4e70439330122a5f0fbac8251101ee92126938c6 Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Fri, 30 Dec 2022 15:50:32 +0100
Subject: [PATCH 47/54] wip: dogstatsd metrics

---
 bin/compile        | 6 +++++-
 bin/supply         | 5 +++++
 lib/run-datadog.sh | 3 +++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/bin/compile b/bin/compile
index 36ea0b2..afb8c39 100755
--- a/bin/compile
+++ b/bin/compile
@@ -62,4 +62,8 @@ chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
-python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
\ No newline at end of file
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
+
+# export DD_TAGS for ddtrace
+export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_DOGSTATSD_TAGS="${DD_TAGS}"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index 6683604..000e363 100755
--- a/bin/supply
+++ b/bin/supply
@@ -63,3 +63,8 @@ printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$
 
 # sanitize env vars and export a new a env file
 python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
+
+# export DD_TAGS for ddtrace
+export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_DOGSTATSD_TAGS="${DD_TAGS}"
+
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 35d2337..d7cd9b4 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -10,6 +10,8 @@ DD_ENABLE_CAPI_METADATA_COLLECTION="${DD_ENABLE_CAPI_METADATA_COLLECTION:-false}
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 USER_TAGS="${DD_TAGS}"
+export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_DOGSTATSD_TAGS="${DD_TAGS}"
 
 source "${DATADOG_DIR}/scripts/utils.sh"
 
@@ -93,6 +95,7 @@ setup_datadog() {
     # Create folder for storing PID files
     mkdir run
 
+
     # DSD requires its own config file
     cp dist/datadog.yaml dist/dogstatsd.yaml
 

From 6b5d01b966353f18389c5c14fcca344f7d53a587 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Fri, 30 Dec 2022 15:41:27 -0500
Subject: [PATCH 48/54] Export dogstatsd tags as space separated

---
 bin/compile                        | 2 +-
 bin/supply                         | 2 +-
 lib/run-datadog.sh                 | 4 ++--
 lib/scripts/get_tags.py            | 2 +-
 lib/scripts/update_agent_config.sh | 3 ++-
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/bin/compile b/bin/compile
index afb8c39..5f585f3 100755
--- a/bin/compile
+++ b/bin/compile
@@ -66,4 +66,4 @@ python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_e
 
 # export DD_TAGS for ddtrace
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS="${DD_TAGS}"
\ No newline at end of file
+export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index 000e363..6a0c37d 100755
--- a/bin/supply
+++ b/bin/supply
@@ -66,5 +66,5 @@ python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_e
 
 # export DD_TAGS for ddtrace
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS="${DD_TAGS}"
+export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
 
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index d7cd9b4..9d577fa 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -11,7 +11,7 @@ LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 USER_TAGS="${DD_TAGS}"
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS="${DD_TAGS}"
+export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
 
 source "${DATADOG_DIR}/scripts/utils.sh"
 
@@ -117,7 +117,7 @@ setup_datadog() {
 start_datadog() {
   DD_TAGS="${USER_TAGS}"
   export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-  export DD_DOGSTATSD_TAGS="${DD_TAGS}"
+  export DD_DOGSTATSD_TAGS="$(python "${DATADOG_DIR}"/scripts/get_tags.py)"
 
   pushd "${DATADOG_DIR}"
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
diff --git a/lib/scripts/get_tags.py b/lib/scripts/get_tags.py
index 8304b92..296faba 100644
--- a/lib/scripts/get_tags.py
+++ b/lib/scripts/get_tags.py
@@ -82,4 +82,4 @@ def parse_tags(tags):
 if legacy_tags:
     print(','.join(tags))
 else:
-    print(json.dumps(tags))
\ No newline at end of file
+    print(' '.join(tags))
\ No newline at end of file
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index bfd092d..4eecaac 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -57,10 +57,10 @@ main() {
 
     log_info "finished check_datadog script"
 
-   
 
     # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
     export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py")"
+    export DD_DOGSTATSD_TAGS="$(python "${DATADOG_DIR}/scripts/get_tags.py")"
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
 
@@ -80,6 +80,7 @@ main() {
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
     log_debug "node_agent_tags.txt=$(cat ${DATADOG_DIR}/node_agent_tags.txt)"
     log_debug "(AFTER)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
+    log_debug "DD_DOGSTATSD_TAGS=${DD_DOGSTATSD_TAGS}"
 
     # finishing up
     log_info "exporting .sourced_datadog_env file"

From 63e269d13f6f313efe3c9de1d1b27420b32b4039 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Fri, 30 Dec 2022 17:16:04 -0500
Subject: [PATCH 49/54] Print correct logs config

---
 lib/scripts/create_logs_config.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index 225a940..2b14659 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -17,7 +17,8 @@
 def sanitize(tags_env_var)
   tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
   tags_list.keep_if { |element| !element.include?(";") }
-  return tags_list
+  tags_list = tags_list.map { |tag| tag.gsub(" ", "_") }
+  return tags_list.uniq
 end
 
 config = {}
@@ -46,7 +47,7 @@ def sanitize(tags_env_var)
 
   if !tags_list.empty?
     tags_list = tags_list.uniq
-    config["logs"][0]["tags"] = tags_list.join(",")
+    config["logs"][0]["tags"] = tags_list
   end
   
 else

From eb8ead075198fd9b7a2182912f4ca8fa3a63f830 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Tue, 3 Jan 2023 16:27:40 -0500
Subject: [PATCH 50/54] Wait for .datadog_env

---
 lib/scripts/update_agent_config.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 4eecaac..6cb2a7e 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -17,6 +17,11 @@ release_lock() {
 
 main() {
     # source relevant DD tags
+    while ! [ -f "${DATADOG_DIR}/.datadog_env" ]; do
+        log_info ".datadog_env file not found, waiting..."
+        sleep 2
+    done
+
     . "${DATADOG_DIR}/.datadog_env"
 
     if [ "${DD_ENABLE_CAPI_METADATA_COLLECTION}" != "true" ]; then

From 1f2d39d3e98603665e2a22a9a032294381d4544b Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Tue, 3 Jan 2023 19:50:29 -0500
Subject: [PATCH 51/54] create supply script completed file

---
 bin/supply                         |  2 +-
 lib/scripts/update_agent_config.sh | 10 +++++-----
 lib/scripts/utils.sh               |  1 -
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/bin/supply b/bin/supply
index 6a0c37d..982235b 100755
--- a/bin/supply
+++ b/bin/supply
@@ -67,4 +67,4 @@ python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_e
 # export DD_TAGS for ddtrace
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
-
+touch "${DATADOG_DIR}/.supply_completed"
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 6cb2a7e..7e0abe1 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -8,7 +8,6 @@ DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 LOCK="${DATADOG_DIR}/update.lock"
 
 # import utils function such as log_message
-. "${DATADOG_DIR}/scripts/utils.sh"
 
 release_lock() {
     log_info "releasing lock '${LOCK}'"
@@ -17,11 +16,12 @@ release_lock() {
 
 main() {
     # source relevant DD tags
-    while ! [ -f "${DATADOG_DIR}/.datadog_env" ]; do
-        log_info ".datadog_env file not found, waiting..."
-        sleep 2
+    while ! [ -f "${DATADOG_DIR}/.supply_completed" ]; do
+        echo "Supply script not completed, waiting ..."
+        sleep 1
     done
 
+    . "${DATADOG_DIR}/scripts/utils.sh"
     . "${DATADOG_DIR}/.datadog_env"
 
     if [ "${DD_ENABLE_CAPI_METADATA_COLLECTION}" != "true" ]; then
@@ -99,4 +99,4 @@ main() {
 }
 
 # for debugging purposes
-main "$@" 2>&1 | tee -a "${DEBUG_FILE}"
+main "$@" 2>&1 | tee -a "${DATADOG_DIR}/update_agent_script.log"
diff --git a/lib/scripts/utils.sh b/lib/scripts/utils.sh
index c08bbd0..79d2a8d 100644
--- a/lib/scripts/utils.sh
+++ b/lib/scripts/utils.sh
@@ -8,7 +8,6 @@
 # https://github.com/DataDog/datadog-agent-boshrelease/blob/4.11.2/src/helpers/lib.sh
 
 export DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
-export DEBUG_FILE="${DATADOG_DIR}/update_agent_script.log"
 export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
 export LOGS_CONFIG
 

From 1e294fc7f04ddb6ffb62b3bed32caa7b82482a0f Mon Sep 17 00:00:00 2001
From: Noueman Khalikine <noueman.khalikine@datadoghq.com>
Date: Wed, 4 Jan 2023 13:49:47 +0100
Subject: [PATCH 52/54] extend script completed file to bin/compile as well +
 small rename of the lockfile to matcht the script name

---
 bin/compile                        | 5 ++++-
 bin/supply                         | 4 +++-
 lib/scripts/update_agent_config.sh | 4 ++--
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/bin/compile b/bin/compile
index 5f585f3..22d2f4f 100755
--- a/bin/compile
+++ b/bin/compile
@@ -66,4 +66,7 @@ python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_e
 
 # export DD_TAGS for ddtrace
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
\ No newline at end of file
+export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+
+# mark the script as finished, useful to sync the update_agent_config script
+touch "${DATADOG_DIR}/.setup_completed"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index 982235b..07b9b65 100755
--- a/bin/supply
+++ b/bin/supply
@@ -67,4 +67,6 @@ python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_e
 # export DD_TAGS for ddtrace
 export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
 export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
-touch "${DATADOG_DIR}/.supply_completed"
+
+# mark the script as finished, useful to sync the update_agent_config script
+touch "${DATADOG_DIR}/.setup_completed"
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 7e0abe1..7ddde6a 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -5,7 +5,7 @@
 # Copyright 2022-Present Datadog, Inc.
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
-LOCK="${DATADOG_DIR}/update.lock"
+LOCK="${DATADOG_DIR}/update_agent_config.lock"
 
 # import utils function such as log_message
 
@@ -16,7 +16,7 @@ release_lock() {
 
 main() {
     # source relevant DD tags
-    while ! [ -f "${DATADOG_DIR}/.supply_completed" ]; do
+    while ! [ -f "${DATADOG_DIR}/.setup_completed" ]; do
         echo "Supply script not completed, waiting ..."
         sleep 1
     done

From 3089405f950a276ae27225daacced2dc8cf78038 Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Wed, 4 Jan 2023 11:19:11 -0500
Subject: [PATCH 53/54] Remove app_instance_guid from node agent tags.txt

---
 lib/scripts/update_tags.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index 4f3b699..35e72bd 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -23,6 +23,7 @@
 def sanitize(tags_env_var)
     tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
     tags_list.keep_if { |element| !element.include?(";") }
+    tags_list.keep_if { |element| !element.include?("app_instance_guid") }
     tags_list = tags_list.map { |tag| tag.gsub(" ", "_") }
     return tags_list.uniq
 end

From 82629561c2e44aeb1b5f4ad3c5119f1f03db519e Mon Sep 17 00:00:00 2001
From: Sarah Witt <sarah.witt@datadoghq.com>
Date: Wed, 4 Jan 2023 21:07:49 -0500
Subject: [PATCH 54/54] Export everything with spaces

---
 bin/compile                        |  7 ++++---
 bin/supply                         |  6 ++++--
 lib/run-datadog.sh                 | 13 ++++++++-----
 lib/scripts/create_logs_config.rb  |  8 ++++----
 lib/scripts/update_agent_config.sh |  8 +++++---
 lib/scripts/update_tags.rb         |  8 ++++----
 6 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/bin/compile b/bin/compile
index 22d2f4f..42777a6 100755
--- a/bin/compile
+++ b/bin/compile
@@ -65,8 +65,9 @@ printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$
 python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
 
 # export DD_TAGS for ddtrace
-export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
-
+DD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_TAGS
+DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_DOGSTATSD_TAGS
 # mark the script as finished, useful to sync the update_agent_config script
 touch "${DATADOG_DIR}/.setup_completed"
\ No newline at end of file
diff --git a/bin/supply b/bin/supply
index 07b9b65..b08c04b 100755
--- a/bin/supply
+++ b/bin/supply
@@ -65,8 +65,10 @@ printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "$
 python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
 
 # export DD_TAGS for ddtrace
-export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+DD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_TAGS
+DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_DOGSTATSD_TAGS
 
 # mark the script as finished, useful to sync the update_agent_config script
 touch "${DATADOG_DIR}/.setup_completed"
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
index 9d577fa..0d69802 100644
--- a/lib/run-datadog.sh
+++ b/lib/run-datadog.sh
@@ -10,8 +10,10 @@ DD_ENABLE_CAPI_METADATA_COLLECTION="${DD_ENABLE_CAPI_METADATA_COLLECTION:-false}
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
 USER_TAGS="${DD_TAGS}"
-export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-export DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+DD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_TAGS
+DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+export DD_DOGSTATSD_TAGS
 
 source "${DATADOG_DIR}/scripts/utils.sh"
 
@@ -116,9 +118,10 @@ setup_datadog() {
 
 start_datadog() {
   DD_TAGS="${USER_TAGS}"
-  export DD_TAGS=$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}"/scripts/get_tags.py)
-  export DD_DOGSTATSD_TAGS="$(python "${DATADOG_DIR}"/scripts/get_tags.py)"
-
+  DD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+  export DD_TAGS
+  DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+  export DD_DOGSTATSD_TAGS
   pushd "${DATADOG_DIR}"
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
     export DD_API_KEY
diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
index 2b14659..d53c491 100644
--- a/lib/scripts/create_logs_config.rb
+++ b/lib/scripts/create_logs_config.rb
@@ -14,8 +14,8 @@
 dd_tags = ENV['DD_TAGS']
 dd_node_agent_tags = ENV['DD_NODE_AGENTS_TAGS'] || (File.file?(node_agent_tags) ? File.read(node_agent_tags) : "")
 
-def sanitize(tags_env_var)
-  tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
+def sanitize(tags_env_var, separator)
+  tags_list = tags_env_var.gsub(",\"", ";\"").split(separator)
   tags_list.keep_if { |element| !element.include?(";") }
   tags_list = tags_list.map { |tag| tag.gsub(" ", "_") }
   return tags_list.uniq
@@ -34,13 +34,13 @@ def sanitize(tags_env_var)
   tags_list = []
 
   if !dd_tags.nil?
-    tags_list += sanitize(dd_tags)
+    tags_list += sanitize(dd_tags, " ")
   else
     puts "Could not find DD_TAGS env var"
   end
 
   if !dd_node_agent_tags.nil?
-    tags_list += sanitize(dd_node_agent_tags)
+    tags_list += sanitize(dd_node_agent_tags, ",")
   else
     puts "Could not find DD_NODE_AGENTS_TAGS env var"
   end
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
index 7ddde6a..f49779c 100644
--- a/lib/scripts/update_agent_config.sh
+++ b/lib/scripts/update_agent_config.sh
@@ -8,7 +8,6 @@ DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 LOCK="${DATADOG_DIR}/update_agent_config.lock"
 
 # import utils function such as log_message
-
 release_lock() {
     log_info "releasing lock '${LOCK}'"
     rmdir "${LOCK}"
@@ -64,8 +63,11 @@ main() {
 
 
     # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
-    export DD_TAGS="$(LEGACY_TAGS_FORMAT=true python "${DATADOG_DIR}/scripts/get_tags.py")"
-    export DD_DOGSTATSD_TAGS="$(python "${DATADOG_DIR}/scripts/get_tags.py")"
+    DD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+    export DD_TAGS
+    DD_DOGSTATSD_TAGS=$(python "${DATADOG_DIR}"/scripts/get_tags.py)
+    export DD_DOGSTATSD_TAGS
+
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
 
diff --git a/lib/scripts/update_tags.rb b/lib/scripts/update_tags.rb
index 35e72bd..5589db8 100644
--- a/lib/scripts/update_tags.rb
+++ b/lib/scripts/update_tags.rb
@@ -20,8 +20,8 @@
 # storing all tags on this variable
 tags = []
 
-def sanitize(tags_env_var)
-    tags_list = tags_env_var.gsub(",\"", ";\"").split(",")
+def sanitize(tags_env_var, separator)
+    tags_list = tags_env_var.gsub(",\"", ";\"").split(separator)
     tags_list.keep_if { |element| !element.include?(";") }
     tags_list.keep_if { |element| !element.include?("app_instance_guid") }
     tags_list = tags_list.map { |tag| tag.gsub(" ", "_") }
@@ -29,11 +29,11 @@ def sanitize(tags_env_var)
 end
 
 if ! DD_NODE_AGENT_TAGS.empty?
-    tags.concat(sanitize(DD_NODE_AGENT_TAGS))
+    tags.concat(sanitize(DD_NODE_AGENT_TAGS, ","))
 end
 
 if ! DD_TAGS.empty?
-    tags.concat(sanitize(DD_TAGS))
+    tags.concat(sanitize(DD_TAGS, " "))
 end
 
 # if the script is executed during the warmup period, merge incoming tags with the existing tags