Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More verbose logging in ssh service #2866

Merged
merged 1 commit into from
Oct 15, 2021
+30 −10
Merged

More verbose logging in ssh service #2866

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 30 additions & 10 deletions services/ssh/home/rsh.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ WAIT_TO_UNIDLE_SERVICES=${WAIT_TO_UNIDLE_SERVICES:-false}
# set a timeout of 600 for waiting for a pod to start (the waits are 1 second interval, so 10 minutes timeout)
SSH_CHECK_TIMEOUT=${SSH_CHECK_TIMEOUT:-600}

# generate a random uuid for this request to help track in logs
# also the uuid will be given to users in any errors so they can provide it to help with tracking too if required
# which makes going through logs easier
UUID=$(cat /proc/sys/kernel/random/uuid)

# get the graphql endpoint, if set
eval "$(grep GRAPHQL_ENDPOINT /authorize.env)"

Expand All @@ -30,11 +35,11 @@ if [[ -n "$REQUESTED_PROJECT" ]]; then
if [[ "$REQUESTED_PROJECT" =~ ^[A-Za-z0-9-]+$ ]]; then
PROJECT=$REQUESTED_PROJECT
else
echo "ERROR: given project '$REQUESTED_PROJECT' contains illegal characters";
echo "${UUID}: ERROR: given project '$REQUESTED_PROJECT' contains illegal characters";
exit 1
fi
else
echo "ERROR: no project defined";
echo "${UUID}: ERROR: no project defined";
exit 1
fi

Expand All @@ -54,7 +59,7 @@ ENVIRONMENT=$(curl -s -XPOST -H 'Content-Type: application/json' -H "$BEARER" "$

# Check if the returned OpenShift projectname is the same as the one being requested. This will only be true if the user actually has access to this environment
if [[ ! "$(echo $ENVIRONMENT | jq --raw-output '.data.userCanSshToEnvironment.openshiftProjectName')" == "$PROJECT" ]]; then
echo "no access to $PROJECT"
echo "${UUID}: no access to $PROJECT"
exit 1
fi

Expand Down Expand Up @@ -97,7 +102,7 @@ else
SERVICE=cli
fi

echo "Incoming Remote Shell Connection: project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}' container='${CONTAINER}' command='$*'" >> /proc/1/fd/1
echo "${UUID}: Incoming Remote Shell Connection: project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}' container='${CONTAINER}' command='$*'" >> /proc/1/fd/1

# This only happens on local development with minishift.
# Login as developer:deveeloper and get the token
Expand All @@ -113,7 +118,7 @@ if [[ $($OC get deploymentconfigs -l service=${SERVICE} 2> /dev/null) ]]; then
DEPLOYMENTCONFIG=$($OC get deploymentconfigs -l service=${SERVICE} -o name)
# If the deploymentconfig is scaled to 0, scale to 1
if [[ $($OC get ${DEPLOYMENTCONFIG} -o go-template --template='{{.status.replicas}}') == "0" ]]; then

echo "${UUID}: Attempting to scale deploymentconfig='${DEPLOYMENTCONFIG}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
$OC scale --replicas=1 ${DEPLOYMENTCONFIG} >/dev/null 2>&1

# Wait until the scaling is done
Expand All @@ -122,6 +127,7 @@ if [[ $($OC get deploymentconfigs -l service=${SERVICE} 2> /dev/null) ]]; then
sleep 1
done
fi
echo "${UUID}: Deployment is running deploymentconfig='${DEPLOYMENTCONFIG}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
fi

# If there is a deployment for the given service searching for lagoon.sh labels
Expand All @@ -131,6 +137,7 @@ if [[ $($OC get deployment -l "lagoon.sh/service=${SERVICE}" 2> /dev/null) ]]; t
# we do this first to give the services a bit of time to unidle before starting the one that was requested
DEPLOYMENTS=$($OC get deployments -l "idling.amazee.io/watch=true" -o name)
if [ ! -z "${DEPLOYMENTS}" ]; then
echo "${UUID}: Environment is idled attempting to scale up for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
# loop over the deployments and unidle them
for DEP in ${DEPLOYMENTS}
do
Expand All @@ -141,6 +148,7 @@ if [[ $($OC get deployment -l "lagoon.sh/service=${SERVICE}" 2> /dev/null) ]]; t
if [ ! -z "$REPLICAS" ]; then
REPLICAS=1
fi
echo "${UUID}: Attempting to scale deployment='${DEP}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
$OC scale --replicas=${REPLICAS} ${DEP} >/dev/null 2>&1
fi
done
Expand All @@ -152,17 +160,19 @@ if [[ $($OC get deployment -l "lagoon.sh/service=${SERVICE}" 2> /dev/null) ]]; t
# WAIT_TO_UNIDLE_SERVICES will default to false so that it just scales the deployments
# and won't wait for them to be ready, but if set to true, it will wait for `readyReplicas` to be 1
if [[ "$WAIT_TO_UNIDLE_SERVICES" =~ [Tt][Rr][Uu][Ee] ]]; then
echo "${UUID}: Environment is idled waiting for scale up for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
SSH_CHECK_COUNTER=0
until [[ $($OC get ${DEP} -o json | jq -r '.status.readyReplicas // 0') -ne "0" ]]
do
if [ $SSH_CHECK_COUNTER -lt $SSH_CHECK_TIMEOUT ]; then
let SSH_CHECK_COUNTER=SSH_CHECK_COUNTER+1
sleep 1
else
echo "Deployment '${DEP}' took too long to start pods"
echo "${UUID}: Deployment '${DEP}' took too long to start pods"
exit 1
fi
done
echo "${UUID}: Environment scaled up for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
fi
done
fi
Expand All @@ -173,6 +183,7 @@ if [[ $($OC get deployment -l "lagoon.sh/service=${SERVICE}" 2> /dev/null) ]]; t
# If the deployment is scaled to 0, scale to 1
# .status.replicas doesn't exist on a scaled to 0 deployment in k8s so assume it is 0 if nothing is returned
if [[ $($OC get ${DEPLOYMENT} -o json | jq -r '.status.replicas // 0') == "0" ]]; then
echo "${UUID}: Attempting to scale deployment='${DEPLOYMENT}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
$OC scale --replicas=1 ${DEPLOYMENT} >/dev/null 2>&1
fi
# Wait until the scaling is done
Expand All @@ -183,10 +194,11 @@ if [[ $($OC get deployment -l "lagoon.sh/service=${SERVICE}" 2> /dev/null) ]]; t
let SSH_CHECK_COUNTER=SSH_CHECK_COUNTER+1
sleep 1
else
echo "Pod for ${SERVICE} took too long to start"
echo "${UUID}: Pod for ${SERVICE} took too long to start"
exit 1
fi
done
echo "${UUID}: Deployment is running deployment='${DEPLOYMENT}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
fi

# If there is a deployment for the given service search for lagoon labels
Expand All @@ -197,6 +209,7 @@ if [[ $($OC get deployment -l lagoon/service=${SERVICE} 2> /dev/null) ]]; then
# we do this first to give the services a bit of time to unidle before starting the one that was requested
DEPLOYMENTS=$($OC get deployments -l "idling.amazee.io/watch=true" -o name)
if [ ! -z "${DEPLOYMENTS}" ]; then
echo "${UUID}: Environment is idled waiting for scale up for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
# loop over the deployments and unidle them
for DEP in ${DEPLOYMENTS}
do
Expand All @@ -207,6 +220,7 @@ if [[ $($OC get deployment -l lagoon/service=${SERVICE} 2> /dev/null) ]]; then
if [ ! -z "$REPLICAS" ]; then
REPLICAS=1
fi
echo "${UUID}: Attempting to scale deployment='${DEP}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
$OC scale --replicas=${REPLICAS} ${DEP} >/dev/null 2>&1
fi
done
Expand All @@ -218,17 +232,19 @@ if [[ $($OC get deployment -l lagoon/service=${SERVICE} 2> /dev/null) ]]; then
# WAIT_TO_UNIDLE_SERVICES will default to false so that it just scales the deployments
# and won't wait for them to be ready, but if set to true, it will wait for `readyReplicas` to be 1
if [[ "$WAIT_TO_UNIDLE_SERVICES" =~ [Tt][Rr][Uu][Ee] ]]; then
echo "${UUID}: Environment is idled waiting for scale up for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
SSH_CHECK_COUNTER=0
until [[ $($OC get ${DEP} -o json | jq -r '.status.readyReplicas // 0') -ne "0" ]]
do
if [ $SSH_CHECK_COUNTER -lt $SSH_CHECK_TIMEOUT ]; then
let SSH_CHECK_COUNTER=SSH_CHECK_COUNTER+1
sleep 1
else
echo "Deployment '${DEP}' took too long to start pods"
echo "${UUID}: Deployment '${DEP}' took too long to start pods"
exit 1
fi
done
echo "${UUID}: Environment scaled up for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
fi
done
fi
Expand All @@ -239,6 +255,7 @@ if [[ $($OC get deployment -l lagoon/service=${SERVICE} 2> /dev/null) ]]; then
# If the deployment is scaled to 0, scale to 1
# .status.replicas doesn't exist on a scaled to 0 deployment in k8s so assume it is 0 if nothing is returned
if [[ $($OC get ${DEPLOYMENT} -o json | jq -r '.status.replicas // 0') == "0" ]]; then
echo "${UUID}: Attempting to scale up deployment='${DEPLOYMENT}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
$OC scale --replicas=1 ${DEPLOYMENT} >/dev/null 2>&1
fi
# Wait until the scaling is done
Expand All @@ -249,13 +266,15 @@ if [[ $($OC get deployment -l lagoon/service=${SERVICE} 2> /dev/null) ]]; then
let SSH_CHECK_COUNTER=SSH_CHECK_COUNTER+1
sleep 1
else
echo "Pod for ${SERVICE} took too long to start"
echo "${UUID}: Pod for ${SERVICE} took too long to start"
exit 1
fi
done
echo "${UUID}: Deployment is running deployment='${DEPLOYMENT}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
fi


echo "${UUID}: Getting pod name for exec for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
POD=$($OC get pods -l service=${SERVICE} -o json | jq -r '[.items[] | select(.metadata.deletionTimestamp == null) | select(.status.phase == "Running")] | first | .metadata.name // empty')

# Check for newer Helm chart "lagoon.sh" labels
Expand All @@ -270,7 +289,7 @@ if [[ ! $POD ]]; then
fi

if [[ ! $POD ]]; then
echo "No running pod found for service ${SERVICE}"
echo "${UUID}: No running pod found for service ${SERVICE}"
exit 1
fi

Expand All @@ -285,6 +304,7 @@ else
TTY_PARAMETER=""
fi

echo "${UUID}: Exec to pod='${POD}' for project='${PROJECT}' openshift='${OPENSHIFT_NAME}' service='${SERVICE}'" >> /proc/1/fd/1
if [[ -z "$*" ]]; then
exec $OC exec ${POD} -c ${CONTAINER} -i ${TTY_PARAMETER} -- sh
else
Expand Down