Skip to content

Commit

Permalink
Merge pull request #105 from turkenh/integration-test-improvements
Browse files Browse the repository at this point in the history 
Integration test fixes and improvements
  • Loading branch information
@turkenh
turkenh authored Sep 6, 2021
2 parents 003a021 + 1b5e3c8 commit bca01f3
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 135 deletions.
5 changes: 3 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ GO111MODULE = on

# ====================================================================================
# Setup Kubernetes tools
KIND_VERSION = v0.7.0
KIND_VERSION ?= v0.11.1
KIND_NODE_IMAGE_TAG ?= v1.19.11
USE_HELM3 = true
-include build/makelib/k8s_tools.mk

Expand Down Expand Up @@ -81,7 +82,7 @@ e2e.run: test-integration
# Run integration tests.
test-integration: $(KIND) $(KUBECTL) $(HELM3)
@$(INFO) running integration tests using kind $(KIND_VERSION)
@$(ROOT_DIR)/cluster/integration/integration_tests.sh || $(FAIL)
@KIND_NODE_IMAGE_TAG=${KIND_NODE_IMAGE_TAG} $(ROOT_DIR)/cluster/integration/integration_tests.sh || $(FAIL)
@$(OK) integration tests passed

# Update the submodules, such as the common build scripts.
Expand Down
2 changes: 1 addition & 1 deletion build
Submodule build updated 5 files
+1 −1 cross/Dockerfile
+1 −2 makelib/common.mk
+9 −1 makelib/docs.mk
+1 −0 makelib/helm.mk
+2 −2 makelib/k8s_tools.mk
150 changes: 18 additions & 132 deletions cluster/integration/integration_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,103 +32,6 @@ echo_error(){
exit 1
}

# k8s watchers

wait_for_pods_in_namespace(){
local timeout=$1
shift
namespace=$1
shift
arr=("$@")
local counter=0
for i in "${arr[@]}";
do
echo -n "waiting for pod $i in namespace $namespace..." >&2
while ! ("${KUBECTL}" -n $namespace get pod $i) &>/dev/null; do
if [ "$counter" -ge "$timeout" ]; then echo "TIMEOUT"; exit -1; else (( counter+=5 )); fi
echo -n "." >&2
sleep 5
done
echo "FOUND POD!" >&2
done
}

check_deployments(){
for name in $1; do
echo_sub_step "inspecting deployment '${name}'"
local dep_stat=$("${KUBECTL}" -n "$2" get deployments/"${name}")

echo_info "check if is deployed"
if $(echo "$dep_stat" | grep -iq 'No resources found'); then
echo "is not deployed"
exit -1
else
echo_step_completed
fi

echo_info "check if is ready"
IFS='/' read -ra ready_status_parts <<< "$(echo "$dep_stat" | awk ' FNR > 1 {print $2}')"
if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
echo "is not Ready"
exit -1
else
echo_step_completed
fi
echo
done
}

check_pods(){
pods=$("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods)
count=$(echo "$pods" | wc -l)
if (("${count}"-1 != "${1}")); then
sleep 10
"${KUBECTL}" get events -A
sleep 20
echo_error "unexpected number of pods"
exit -1
fi
echo "$pods"
while read -r pod_stat; do
name=$(echo "$pod_stat" | awk '{print $1}')
echo_sub_step "inspecting pod '${name}'"

if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Completed'); then
echo_info "is not completed, continuing with further checks"
else
echo_info "is completed, foregoing further checks"
echo_step_completed
continue
fi

echo_info "check if is ready"
IFS='/' read -ra ready_status_parts <<< "$(echo "$pod_stat" | awk '{print $2}')"
if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
echo_error "is not ready"
exit -1
else
echo_step_completed
fi

echo_info "check if is running"
if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Running'); then
echo_error "is not running"
exit -1
else
echo_step_completed
fi

echo_info "check if has restarts"
if (( $(echo "$pod_stat" | awk '{print $4}') > 0 )); then
echo_error "has restarts"
exit -1
else
echo_step_completed
fi
echo
done <<< "$(echo "$pods" | awk 'FNR>1')"
}

# ------------------------------
projectdir="$( cd "$( dirname "${BASH_SOURCE[0]}")"/../.. && pwd )"

Expand All @@ -144,7 +47,7 @@ CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-controller-${SAFEHOSTARCH}"
version_tag="$(cat ${projectdir}/_output/version)"
# tag as latest version to load into kind cluster
PACKAGE_CONTROLLER_IMAGE="${DOCKER_REGISTRY}/${PROJECT_NAME}-controller:${VERSION}"
K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-INTTESTS}"
K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-inttests}"

CROSSPLANE_NAMESPACE="crossplane-system"
PACKAGE_NAME="provider-helm"
Expand All @@ -168,7 +71,8 @@ echo "created cache dir at ${CACHE_PATH}"
docker save "${BUILD_IMAGE}" -o "${CACHE_PATH}/${PACKAGE_NAME}.xpkg" && chmod 644 "${CACHE_PATH}/${PACKAGE_NAME}.xpkg"

# create kind cluster with extra mounts
echo_step "creating k8s cluster using kind"
KIND_NODE_IMAGE="kindest/node:${KIND_NODE_IMAGE_TAG}"
echo_step "creating k8s cluster using kind ${KIND_VERSION} and node image ${KIND_NODE_IMAGE}"
KIND_CONFIG="$( cat <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
Expand All @@ -179,17 +83,12 @@ nodes:
containerPath: /cache
EOF
)"
echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --config=-
echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${KIND_NODE_IMAGE}" --config=-

# tag controller image and load it into kind cluster
docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_CONTROLLER_IMAGE}"
"${KIND}" load docker-image "${PACKAGE_CONTROLLER_IMAGE}" --name="${K8S_CLUSTER}"

# wait for kind pods
echo_step "wait for kind pods"
kindpods=("kube-apiserver-${BUILD_REGISTRY}-inttests-control-plane" "kube-controller-manager-${BUILD_REGISTRY}-inttests-control-plane" "kube-scheduler-${BUILD_REGISTRY}-inttests-control-plane")
wait_for_pods_in_namespace 120 "kube-system" "${kindpods[@]}"

# files are not synced properly from host to kind node container on Jenkins, so
# we must manually copy image from host to node
echo_step "pre-cache package by copying to kind node"
Expand Down Expand Up @@ -254,18 +153,6 @@ echo_step "wait until the pods are up and running"

# ----------- integration tests
echo_step "--- INTEGRATION TESTS ---"
echo
echo_step "check for necessary deployment statuses"
echo
echo -------- deployments
"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments

check_deployments "crossplane" "${CROSSPLANE_NAMESPACE}"

echo_step "check for crossplane pods statuses"
echo
echo "--- pods ---"
check_pods 2

# install package
echo_step "installing ${PROJECT_NAME} into \"${CROSSPLANE_NAMESPACE}\" namespace"
Expand All @@ -283,19 +170,13 @@ EOF

echo "${INSTALL_YAML}" | "${KUBECTL}" apply -f -

"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments

# this is to let package manager unpack pods and start the controller.
sleep 20

# printing the cache dir contents can be useful for troubleshooting failures
echo_step "check kind node cache dir contents"
docker exec "${K8S_CLUSTER}-control-plane" ls -la /cache

echo_step "check for package pod statuses"
echo
echo "--- pods ---"
check_pods 3
echo_step "waiting for provider to be installed"

kubectl wait "provider.pkg.crossplane.io/${PACKAGE_NAME}" --for=condition=healthy --timeout=60s

echo_step "setup provider"
SA=$("${KUBECTL}" -n crossplane-system get sa -o name | grep provider-helm | sed -e 's|serviceaccount\/|crossplane-system:|g')
Expand All @@ -316,17 +197,22 @@ else
fi

echo_sub_step "check release deployment"
check_deployments "wordpress-example" "wordpress"

echo_step "uninstalling ${PROJECT_NAME}"

echo "${INSTALL_YAML}" | "${KUBECTL}" delete -f -

# check pods deleted
sleep 30
echo_step "check only crossplane pods remain"
echo "--- pods ---"
"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods
check_pods 2
timeout=60
current=0
step=3
while [[ $(kubectl get providerrevision.pkg.crossplane.io -o name | wc -l) != "0" ]]; do
echo "waiting for provider to be deleted for another $step seconds"
current=$current+$step
if ! [[ $timeout > $current ]]; then
echo_error "timeout of ${timeout}s has been reached"
fi
sleep $step;
done

echo_success "Integration tests succeeded!"

0 comments on commit bca01f3

Please sign in to comment.