Improve stability of the CAPI metadata tags collection feature

bin/compile

@@ -4,7 +4,7 @@ echo "-----> DatadogBuildpack/compile"
 
 
 BIN_DIR=$(cd $(dirname $0); pwd)
-ROOT_DIR=$(dirname $BIN_DIR)
+ROOT_DIR=$(dirname ${BIN_DIR})
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
@@ -31,12 +31,15 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
+cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
+cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
+
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
 cp "${ROOT_DIR}/lib/run-datadog.sh" "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 cp "${ROOT_DIR}/lib/redirect-logs.sh" "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
@@ -51,12 +54,14 @@ fi
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 chmod +x "${DATADOG_DIR}/trace-agent"
 chmod +x "${BUILD_DIR}/.profile.d/00-test-endpoint.sh"
 chmod +x "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
 chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
 # Export all environment variables to a file that scripts can use later
 # We need to handle quotes and comments; sed command found in https://gist.github.com/mihow/9c7f559807069a03e302605691f85572
-printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${DATADOG_DIR}/.datadog_env"
+printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
+
+# sanitize env vars and export a new a env file
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"

bin/supply

@@ -4,7 +4,7 @@ echo "-----> DatadogBuildpack/supply"
 
 
 BIN_DIR=$(cd $(dirname $0); pwd)
-ROOT_DIR=$(dirname $BIN_DIR)
+ROOT_DIR=$(dirname ${BIN_DIR})
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
@@ -31,12 +31,15 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/create_logs_config.py"
+cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_yaml_config.rb" "${DATADOG_DIR}/scripts/update_yaml_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
+
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
 cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
 cp "${ROOT_DIR}/lib/run-datadog.sh" "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 cp "${ROOT_DIR}/lib/redirect-logs.sh" "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
@@ -51,12 +54,14 @@ fi
 chmod +x "${DATADOG_DIR}/scripts/utils.sh"
 chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"
 
-chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
 chmod +x "${DATADOG_DIR}/trace-agent"
 chmod +x "${BUILD_DIR}/.profile.d/00-test-endpoint.sh"
 chmod +x "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
 chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
 # Export all environment variables to a file that scripts can use later
 # We need to handle quotes and comments; sed command found in https://gist.github.com/mihow/9c7f559807069a03e302605691f85572
-printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${DATADOG_DIR}/.datadog_env"
+printenv | sed -e '/^#/d;/^\s*$/d' -e "s/'/'\\\''/g" -e "s/=\(.*\)/='\1'/g" > "${BUILD_DIR}/.datadog/.raw_datadog_env"
+
+# sanitize env vars and export a new a env file
+python "${DATADOG_DIR}/scripts/parse_env_vars.py" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"

build

@@ -3,89 +3,89 @@ set -e
 
 SRCDIR=$(cd "$(dirname $0)/." && pwd)
 NAME="datadog-cloudfoundry-buildpack"
-ZIPFILE="$NAME.zip"
+ZIPFILE="${NAME}.zip"
 DOWNLOAD_BASE_URL="https://s3.amazonaws.com/apt.datadoghq.com/pool/d/da/datadog-"
-TRACEAGENT_DOWNLOAD_URL=$DOWNLOAD_BASE_URL"agent_"
-IOT_AGENT_DOWNLOAD_URL=$DOWNLOAD_BASE_URL"iot-agent_"
-DOGSTATSD_DOWNLOAD_URL=$DOWNLOAD_BASE_URL"dogstatsd_"
+TRACEAGENT_DOWNLOAD_URL=${DOWNLOAD_BASE_URL}"agent_"
+IOT_AGENT_DOWNLOAD_URL=${DOWNLOAD_BASE_URL}"iot-agent_"
+DOGSTATSD_DOWNLOAD_URL=${DOWNLOAD_BASE_URL}"dogstatsd_"
 DOWNLOAD_URL_TAIL="-1_amd64.deb"
 AGENT_DEFAULT_VERSION="7.40.1"
 
-TMPDIR="$SRCDIR/tmp"
+TMPDIR="${SRCDIR}/tmp"
 
 function download_trace_agent() {
-  local trace_version="${1:-$AGENT_DEFAULT_VERSION}"
-  local trace_agent_download_url="$TRACEAGENT_DOWNLOAD_URL$trace_version$DOWNLOAD_URL_TAIL"
+  local trace_version="${1:-${AGENT_DEFAULT_VERSION}}"
+  local trace_agent_download_url="${TRACEAGENT_DOWNLOAD_URL}${trace_version}${DOWNLOAD_URL_TAIL}"
 
-  mkdir -p $TMPDIR
-  curl -L $trace_agent_download_url -o ./tmp/datadog-agent.deb
-  pushd $TMPDIR
+  mkdir -p ${TMPDIR}
+  curl -L ${trace_agent_download_url} -o ./tmp/datadog-agent.deb
+  pushd ${TMPDIR}
     dpkg -x datadog-agent.deb .
   popd
-  cp $TMPDIR/opt/datadog-agent/embedded/bin/trace-agent $SRCDIR/lib/trace-agent
-  rm -rf $TMPDIR/*
+  cp ${TMPDIR}/opt/datadog-agent/embedded/bin/trace-agent ${SRCDIR}/lib/trace-agent
+  rm -rf ${TMPDIR}/*
 }
 
 function download_iot_agent() {
-  local iot_version="${1:-$AGENT_DEFAULT_VERSION}"
-  local iot_agent_download_url="$IOT_AGENT_DOWNLOAD_URL$iot_version$DOWNLOAD_URL_TAIL"
+  local iot_version="${1:-${AGENT_DEFAULT_VERSION}}"
+  local iot_agent_download_url="${IOT_AGENT_DOWNLOAD_URL}${iot_version}${DOWNLOAD_URL_TAIL}"
 
-  mkdir -p $TMPDIR
-  curl -L $iot_agent_download_url -o ./tmp/datadog-agent.deb
-  pushd $TMPDIR
+  mkdir -p ${TMPDIR}
+  curl -L ${iot_agent_download_url} -o ./tmp/datadog-agent.deb
+  pushd ${TMPDIR}
     dpkg -x datadog-agent.deb .
   popd
-  cp $TMPDIR/opt/datadog-agent/bin/agent/agent $SRCDIR/lib/agent
-  rm -rf $TMPDIR/*
+  cp ${TMPDIR}/opt/datadog-agent/bin/agent/agent ${SRCDIR}/lib/agent
+  rm -rf ${TMPDIR}/*
 }
 
 function download_dogstatsd() {
-  local dogstatsd_version="${1:-$AGENT_DEFAULT_VERSION}"
-  local dogstatsd_download_url="$DOGSTATSD_DOWNLOAD_URL$dogstatsd_version$DOWNLOAD_URL_TAIL"
+  local dogstatsd_version="${1:-${AGENT_DEFAULT_VERSION}}"
+  local dogstatsd_download_url="${DOGSTATSD_DOWNLOAD_URL}${dogstatsd_version}${DOWNLOAD_URL_TAIL}"
 
-  mkdir -p $TMPDIR
-  curl -L $dogstatsd_download_url -o ./tmp/dogstatsd.deb
-  pushd $TMPDIR
+  mkdir -p ${TMPDIR}
+  curl -L ${dogstatsd_download_url} -o ./tmp/dogstatsd.deb
+  pushd ${TMPDIR}
     dpkg -x dogstatsd.deb .
   popd
-  cp $TMPDIR/opt/datadog-dogstatsd/bin/dogstatsd $SRCDIR/lib/dogstatsd
-  rm -rf $TMPDIR/*
+  cp ${TMPDIR}/opt/datadog-dogstatsd/bin/dogstatsd ${SRCDIR}/lib/dogstatsd
+  rm -rf ${TMPDIR}/*
 }
 
 function main() {
-  if [ ! -f $SRCDIR/lib/dogstatsd ] || [ ! -f $SRCDIR/lib/trace-agent ]; then
+  if [ ! -f ${SRCDIR}/lib/dogstatsd ] || [ ! -f ${SRCDIR}/lib/trace-agent ]; then
     DOWNLOAD="true"
   fi
-  if [ -n "$IOT_AGENT" ] && [ ! -f $SRCDIR/lib/agent ]; then
+  if [ -n "${IOT_AGENT}" ] && [ ! -f ${SRCDIR}/lib/agent ]; then
     DOWNLOAD="true"
   fi
-  if [ -n "$REFRESH_ASSETS" ]; then
+  if [ -n "${REFRESH_ASSETS}" ]; then
     DOWNLOAD="true"
   fi
-  if [ -n "$DOWNLOAD" ]; then
+  if [ -n "${DOWNLOAD}" ]; then
     # Delete the old ones
-    rm -f $SRCDIR/lib/agent
-    rm -f $SRCDIR/lib/dogstatsd
-    rm -f $SRCDIR/lib/trace-agent
+    rm -f ${SRCDIR}/lib/agent
+    rm -f ${SRCDIR}/lib/dogstatsd
+    rm -f ${SRCDIR}/lib/trace-agent
 
     # Download the new ones
-    VERSION=${VERSION:-$AGENT_DEFAULT_VERSION}
+    VERSION=${VERSION:-${AGENT_DEFAULT_VERSION}}
 
-    download_trace_agent $VERSION
-    chmod +x $SRCDIR/lib/trace-agent
+    download_trace_agent ${VERSION}
+    chmod +x ${SRCDIR}/lib/trace-agent
 
-    download_iot_agent $VERSION
-    chmod +x $SRCDIR/lib/agent
+    download_iot_agent ${VERSION}
+    chmod +x ${SRCDIR}/lib/agent
 
-    download_dogstatsd $VERSION
-    chmod +x $SRCDIR/lib/dogstatsd
+    download_dogstatsd ${VERSION}
+    chmod +x ${SRCDIR}/lib/dogstatsd
   fi
 
-  rm -f $ZIPFILE
+  rm -f ${ZIPFILE}
 
-  pushd $SRCDIR
-    if [ ! "$NO_ZIP" ]; then
-      zip -r "$ZIPFILE" lib bin
+  pushd ${SRCDIR}
+    if [ ! "${NO_ZIP}" ]; then
+      zip -r "${ZIPFILE}" lib bin
     fi
   popd
 }

docker/build.sh

@@ -2,18 +2,18 @@
 
 DOCKER_DIR=$(cd "$(dirname $0)/." && pwd)
 
-ROOT_DIR="$DOCKER_DIR/.."
+ROOT_DIR="${DOCKER_DIR}/.."
 
 # First, build the buildpack.
 # The build script is the easiest way to get the buildpack assets
 # This script allows the assets to be grabbed without building the buildpack itself
 
-if [ -z "$DO_NOT_BUILD" ]; then
-  pushd $ROOT_DIR
+if [ -z "${DO_NOT_BUILD}" ]; then
+  pushd ${ROOT_DIR}
     NO_ZIP=true REFRESH_ASSETS=true ./build
   popd
 fi
 
 # Build the container
 
-docker build $ROOT_DIR -t docker_app_with_buildpack -f $DOCKER_DIR/Dockerfile
+docker build ${ROOT_DIR} -t docker_app_with_buildpack -f ${DOCKER_DIR}/Dockerfile

docker/run.sh

@@ -2,10 +2,10 @@
 
 # This is a simple script to run the application.
 # Used only for testing your docker image locally.
-# You will need to have the $DD_API_KEY available in order to run it
+# You will need to have the ${DD_API_KEY} available in order to run it
 
 docker run --rm \
-  -e DD_API_KEY="$DD_API_KEY" \
+  -e DD_API_KEY="${DD_API_KEY}" \
   -e PORT=5050 \
   -p 5050:5050 \
   docker_app_with_buildpack

lib/redirect-logs.sh

@@ -9,48 +9,27 @@ export STD_LOG_COLLECTION_PORT
 
 DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
 
+source "${DATADOG_DIR}/scripts/utils.sh"
+
 DD_EU_API_SITE="https://api.datadoghq.eu/api/"
 DD_US_API_SITE="https://api.datadoghq.com/api/"
-DD_API_SITE=$DD_US_API_SITE
+DD_API_SITE=${DD_US_API_SITE}
 
-if [ -n "$DD_SITE" ] && [ "$DD_SITE" = "datadoghq.eu" ]; then
-  DD_API_SITE=$DD_EU_API_SITE
+if [ -n "${DD_SITE}" ] && [ "${DD_SITE}" = "datadoghq.eu" ]; then
+  DD_API_SITE=${DD_EU_API_SITE}
 fi
 
-# redirect forwards all standard inputs to a TCP socket listening on port STD_LOG_COLLECTION_PORT.
-redirect() {
-  while kill -0 $$; do
-    if [ "$DD_SPARSE_APP_LOGS" = "true" ]; then
-        python "${DATADOG_DIR}/scripts/nc.py" "$STD_LOG_COLLECTION_PORT" || sleep 0.5
-    else
-        nc localhost "$STD_LOG_COLLECTION_PORT" || sleep 0.5
-    fi
-    echo "Resetting buildpack log redirection"
-    if [ "$DD_DEBUG_STD_REDIRECTION" = "true" ]; then
-      HTTP_PROXY=$DD_HTTP_PROXY HTTPS_PROXY=$DD_HTTPS_PROXY NO_PROXY=$DD_NO_PROXY curl \
-      -X POST -H "Content-type: application/json" \
-      -d "{
-            \"title\": \"Resetting buildpack log redirection\",
-            \"text\": \"TCP socket on port $STD_LOG_COLLECTION_PORT for log redirection closed. Restarting it.\",
-            \"priority\": \"normal\",
-            \"tags\": $(python ${DATADOG_DIR}/scripts/get_tags.py),
-            \"alert_type\": \"info\"
-      }" "${DD_API_SITE}v1/events?api_key=$DD_API_KEY"
-    fi
-  done
-}
-
 # setup the redirection from stdout/stderr to the logs-agent.
-if [ "$DD_LOGS_ENABLED" = "true" ]; then
-  if [ "$DD_LOGS_VALID_ENDPOINT" = "false" ]; then
+if [ "${DD_LOGS_ENABLED}" = "true" ]; then
+  if [ "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
     echo "Log endpoint not valid, not starting log redirection"
   else
-    if [ -z "$LOGS_CONFIG" ]; then
+    if [ -z "${LOGS_CONFIG}" ]; then
       echo "can't collect logs, LOGS_CONFIG is not set"
     else
-      echo "collect all logs for config $LOGS_CONFIG"
-      if [ -n "$STD_LOG_COLLECTION_PORT" ]; then
-        echo "forward all logs from stdout/stderr to agent port $STD_LOG_COLLECTION_PORT"
+      echo "collect all logs for config ${LOGS_CONFIG}"
+      if [ -n "${STD_LOG_COLLECTION_PORT}" ]; then
+        echo "forward all logs from stdout/stderr to agent port ${STD_LOG_COLLECTION_PORT}"
         exec &> >(tee >(redirect))
       fi
     fi