Skip to content

Commit

Permalink
Integration test improvements
Browse files Browse the repository at this point in the history 
Following crossplane-contrib/provider-aws#711

Signed-off-by: Hasan Turken <turkenh@gmail.com>
  • Loading branch information
@turkenh
turkenh committed Aug 25, 2021
1 parent 38e34d4 commit 18b7c70
Showing 1 changed file with 13 additions and 137 deletions.
150 changes: 13 additions & 137 deletions cluster/local/integration_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,103 +32,6 @@ echo_error(){
exit 1
}

# k8s watchers

wait_for_pods_in_namespace(){
local timeout=$1
shift
namespace=$1
shift
arr=("$@")
local counter=0
for i in "${arr[@]}";
do
echo -n "waiting for pod $i in namespace $namespace..." >&2
while ! ("${KUBECTL}" -n $namespace get pod $i) &>/dev/null; do
if [ "$counter" -ge "$timeout" ]; then echo "TIMEOUT"; exit -1; else (( counter+=5 )); fi
echo -n "." >&2
sleep 5
done
echo "FOUND POD!" >&2
done
}

check_deployments(){
for name in $1; do
echo_sub_step "inspecting deployment '${name}'"
local dep_stat=$("${KUBECTL}" -n "$2" get deployments/"${name}")

echo_info "check if is deployed"
if $(echo "$dep_stat" | grep -iq 'No resources found'); then
echo "is not deployed"
exit -1
else
echo_step_completed
fi

echo_info "check if is ready"
IFS='/' read -ra ready_status_parts <<< "$(echo "$dep_stat" | awk ' FNR > 1 {print $2}')"
if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
echo "is not Ready"
exit -1
else
echo_step_completed
fi
echo
done
}

check_pods(){
pods=$("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods)
count=$(echo "$pods" | wc -l)
if (("${count}"-1 != "${1}")); then
sleep 10
"${KUBECTL}" get events -A
sleep 20
echo_error "unexpected number of pods"
exit -1
fi
echo "$pods"
while read -r pod_stat; do
name=$(echo "$pod_stat" | awk '{print $1}')
echo_sub_step "inspecting pod '${name}'"

if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Completed'); then
echo_info "is not completed, continuing with further checks"
else
echo_info "is completed, foregoing further checks"
echo_step_completed
continue
fi

echo_info "check if is ready"
IFS='/' read -ra ready_status_parts <<< "$(echo "$pod_stat" | awk '{print $2}')"
if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
echo_error "is not ready"
exit -1
else
echo_step_completed
fi

echo_info "check if is running"
if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Running'); then
echo_error "is not running"
exit -1
else
echo_step_completed
fi

echo_info "check if has restarts"
if (( $(echo "$pod_stat" | awk '{print $4}') > 0 )); then
echo_error "has restarts"
exit -1
else
echo_step_completed
fi
echo
done <<< "$(echo "$pods" | awk 'FNR>1')"
}

# ------------------------------
projectdir="$( cd "$( dirname "${BASH_SOURCE[0]}")"/../.. && pwd )"

Expand Down Expand Up @@ -182,17 +85,12 @@ nodes:
containerPath: /cache
EOF
)"
echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --config=-
echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --config=-

# tag controller image and load it into kind cluster
docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_CONTROLLER_IMAGE}"
"${KIND}" load docker-image "${PACKAGE_CONTROLLER_IMAGE}" --name="${K8S_CLUSTER}"

# wait for kind pods
echo_step "wait for kind pods"
kindpods=("kube-apiserver-${BUILD_REGISTRY}-inttests-control-plane" "kube-controller-manager-${BUILD_REGISTRY}-inttests-control-plane" "kube-scheduler-${BUILD_REGISTRY}-inttests-control-plane")
wait_for_pods_in_namespace 120 "kube-system" "${kindpods[@]}"

# files are not synced properly from host to kind node container on Jenkins, so
# we must manually copy image from host to node
echo_step "pre-cache package by copying to kind node"
Expand Down Expand Up @@ -248,28 +146,10 @@ echo_info "using crossplane version ${chart_version}"
echo
# we replace empty dir with our PVC so that the /cache dir in the kind node
# container is exposed to the crossplane pod
"${HELM3}" install crossplane --namespace crossplane-system crossplane-master/crossplane --version ${chart_version} --devel --set packageCache.pvc=package-cache

echo_step "waiting for deployment crossplane rollout to finish"
"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" rollout status "deploy/crossplane" --timeout=2m

echo_step "wait until the pods are up and running"
"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" wait --for=condition=Ready pods --all --timeout=1m
"${HELM3}" install crossplane --namespace crossplane-system crossplane-master/crossplane --version ${chart_version} --devel --wait --set packageCache.pvc=package-cache

# ----------- integration tests
echo_step "--- INTEGRATION TESTS ---"
echo
echo_step "check for necessary deployment statuses"
echo
echo -------- deployments
"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments

check_deployments "crossplane" "${CROSSPLANE_NAMESPACE}"

echo_step "check for crossplane pods statuses"
echo
echo "--- pods ---"
check_pods 2

# install package
echo_step "installing ${PROJECT_NAME} into \"${CROSSPLANE_NAMESPACE}\" namespace"
Expand Down Expand Up @@ -297,20 +177,10 @@ EOF

echo "${INSTALL_YAML}" | "${KUBECTL}" apply -f -

"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments

# this is to let package manager unpack pods and start the controller.
sleep 20

# printing the cache dir contents can be useful for troubleshooting failures
echo_step "check kind node cache dir contents"
docker exec "${K8S_CLUSTER}-control-plane" ls -la /cache

echo_step "check for package pod statuses"
echo
echo "--- pods ---"
check_pods 3

# install MariaDB chart
echo_step "installing MariaDB Helm chart into default namespace"
mariadb_root_pw=$(LC_ALL=C tr -cd "A-Za-z0-9" </dev/urandom | head -c 32)
Expand Down Expand Up @@ -399,10 +269,16 @@ echo "${PROVIDER_CONFIG_YAML}" | "${KUBECTL}" delete -f -
echo "${INSTALL_YAML}" | "${KUBECTL}" delete -f -

# check pods deleted
sleep 30
echo_step "check only crossplane pods remain"
echo "--- pods ---"
"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods
check_pods 2
timeout=60
current=0
step=3
while [[ $(kubectl get providerrevision.pkg.crossplane.io -o name | wc -l) != "0" ]]; do
echo "waiting for provider to be deleted for another $step seconds"
current=$current+$step
if ! [[ $timeout > $current ]]; then
echo_error "timeout of ${timeout}s has been reached"
fi
sleep $step;
done

echo_success "Integration tests succeeded!"

0 comments on commit 18b7c70

Please sign in to comment.