Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Wait for agent startup before updating CAPI tags #145

Merged
merged 5 commits into from
Dec 8, 2022
+64 −18
Merged

Wait for agent startup before updating CAPI tags #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
19b820e
Properly wait for agent startup
sarah-witt Dec 5, 2022
c43bce7
Wait for token as well
sarah-witt Dec 5, 2022
6f75187
small refactor
NouemanKHAL Dec 7, 2022
a93dc33
Add back datadog dir when using as argument; harmonize compile and su…
sarah-witt Dec 7, 2022
65310a5
Fix pid file name
sarah-witt Dec 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bin/compile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"

cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
Expand All @@ -48,6 +49,7 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
fi

chmod +x "${DATADOG_DIR}/scripts/utils.sh"
chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"

chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
chmod +x "${DATADOG_DIR}/trace-agent"
Expand Down
2 changes: 2 additions & 0 deletions bin/supply
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ cp "${ROOT_DIR}/lib/scripts/create_logs_config.py" "${DATADOG_DIR}/scripts/creat
cp "${ROOT_DIR}/lib/scripts/parse_env_vars.py" "${DATADOG_DIR}/scripts/parse_env_vars.py"
cp "${ROOT_DIR}/lib/scripts/nc.py" "${DATADOG_DIR}/scripts/nc.py"
cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"

cp "${ROOT_DIR}/lib/scripts/update_agent_config_restart.sh" "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
cp -r "${ROOT_DIR}/lib/test-endpoint.sh" "${BUILD_DIR}/.profile.d/00-test-endpoint.sh" # Make sure this is sourced first
Expand All @@ -48,6 +49,7 @@ if [ -f "${DATADOG_DIR}/dogstatsd" ]; then
fi

chmod +x "${DATADOG_DIR}/scripts/utils.sh"
chmod +x "${DATADOG_DIR}/scripts/check_datadog.sh"

chmod +x "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
chmod +x "${DATADOG_DIR}/trace-agent"
Expand Down
Empty file modified build
100644 → 100755
View file
Open in desktop
Empty file.
34 changes: 34 additions & 0 deletions lib/scripts/check_datadog.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env bash

# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
# This product includes software developed at Datadog (https://www.datadoghq.com/).
# Copyright 2022-Present Datadog, Inc.

DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"

check_datadog() {
while true; do
echo "Waiting for agent or dogstatsd process to start"
if [ -f "${DATADOG_DIR}/run/agent.pid" ]; then
echo "Found agent process"
if [ -f "${DATADOG_DIR}/dist/auth_token" ]; then
echo "Found agent token"
break
else
echo "Agent token not found"
fi
fi

if [ -f "${DATADOG_DIR}/run/dogstatsd.pid" ]; then
echo "Found dogstatsd process"
break
fi
sleep 1
done
}

main() {
check_datadog
}

main "$@"
25 changes: 16 additions & 9 deletions lib/scripts/update_agent_config.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
#!/bin/sh

# wait for the buildpack scripts to finish
timeout=0
while { ! [ "$(pgrep -f ./agent)" = "" ] && ! [ "$(pgrep -f ./dogstatsd)" = "" ]; } && [ $timeout -lt 120 ]; do
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since these processes are started using a pid file and it's logically possible for the associated buildpack applications to have a binary with the same name, does use of -F make more sense with pgrep?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it does make more sense. However, when this script is invoked by the Garden API, the underlying process doesn't have the DATADOG_DIR ENV variable, which we would need to find the pid file (e.g.: ${DATADOG_DIR}/run/agent.pid) . @sarah-witt was using this approach as a temporary workaround. We didn't want to assume that DATADOG_DIR=/home/vcap/app/.datadog without careful consideration, as we noticed in some parts in the scripts DATADOG_DIR could be a user input.
We're going to make a separate PR to address this issue and then we'll resume this PR.

sleep 1
timeout=$((timeout+1))
done
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
# This product includes software developed at Datadog (https://www.datadoghq.com/).
# Copyright 2022-Present Datadog, Inc.

# for debugging purposes
echo $DD_NODE_AGENT_TAGS >> /home/vcap/app/.datadog/dd-node-agent-tags.log
DATADOG_DIR="${DATADOG_DIR:-/home/vcap/app/.datadog}"
DEBUG_FILE="${DATADOG_DIR}/update_agent_config_out.log"

main() {
# wait for the buildpack scripts to finish
echo "Starting to wait for agent process to start"
timeout 120s "${DATADOG_DIR}/scripts/check_datadog.sh"

/bin/bash /home/vcap/app/.datadog/scripts/update_agent_config_restart.sh
echo "$DD_NODE_AGENT_TAGS"

/bin/bash "${DATADOG_DIR}/scripts/update_agent_config_restart.sh"
}
# for debugging purposes
main "$@" >> "$DEBUG_FILE" 2>&1
19 changes: 10 additions & 9 deletions lib/scripts/update_agent_config_restart.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,24 @@ stop_datadog() {
if [ -f run/agent.pid ]; then
echo "Stopping agent process, pid: $(cat run/agent.pid)"
(./agent stop --cfgpath dist/) || true
find_pid_kill_and_wait "agent" || true
kill_and_wait "run/agent.pid" 5
agent_commad="./agent run --cfgpath dist/ --pidfile run/agent.pid"
find_pid_kill_and_wait "$agent_commad" || true
kill_and_wait "${DATADOG_DIR}/run/agent.pid" 5
rm -f "run/agent.pid"
fi

if [ -f run/trace.pid ]; then
if [ -f run/trace-agent.pid ]; then
echo "Stopping trace agent process, pid: $(cat run/trace-agent.pid)"
trace_agent_command="trace-agent"
kill_and_wait "run/trace-agent.pid" 5 1
find_pid_kill_and_wait $trace_agent_command "run/trace-agent.pid"
trace_agent_command="./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid"
kill_and_wait "${DATADOG_DIR}/run/trace-agent.pid" 5 1
find_pid_kill_and_wait "$trace_agent_command" "${DATADOG_DIR}/run/trace-agent.pid"
fi

if [ -f run/dogstatsd.pid ]; then
echo "Stopping dogstatsd agent process, pid: $(cat run/dogstatsd.pid)"
dogstatsd_command="dogstatsd"
kill_and_wait "run/dogstatsd.pid" 5 1
find_pid_kill_and_wait $dogstatsd_command "run/dogstatsd.pid"
dogstatsd_command="./dogstatsd start --cfgpath dist/"
kill_and_wait "${DATADOG_DIR}/run/dogstatsd.pid" 5 1
find_pid_kill_and_wait "$dogstatsd_command" "${DATADOG_DIR}/run/dogstatsd.pid"
fi
popd
}
Expand Down