Integration test fixes and improvements

Makefile

@@ -30,7 +30,8 @@ GO111MODULE = on
 
 # ====================================================================================
 # Setup Kubernetes tools
-KIND_VERSION = v0.7.0
+KIND_VERSION ?= v0.11.1
+KIND_NODE_IMAGE_TAG ?= v1.19.11
 USE_HELM3 = true
 -include build/makelib/k8s_tools.mk
 
@@ -81,7 +82,7 @@ e2e.run: test-integration
 # Run integration tests.
 test-integration: $(KIND) $(KUBECTL) $(HELM3)
 	@$(INFO) running integration tests using kind $(KIND_VERSION)
-	@$(ROOT_DIR)/cluster/integration/integration_tests.sh || $(FAIL)
+	@KIND_NODE_IMAGE_TAG=${KIND_NODE_IMAGE_TAG} $(ROOT_DIR)/cluster/integration/integration_tests.sh || $(FAIL)
 	@$(OK) integration tests passed
 
 # Update the submodules, such as the common build scripts.

cluster/integration/integration_tests.sh

@@ -32,103 +32,6 @@ echo_error(){
     exit 1
 }
 
-# k8s watchers
-
-wait_for_pods_in_namespace(){
-    local timeout=$1
-    shift
-    namespace=$1
-    shift
-    arr=("$@")
-    local counter=0
-    for i in "${arr[@]}";
-        do
-            echo -n "waiting for pod $i in namespace $namespace..." >&2
-            while ! ("${KUBECTL}" -n $namespace get pod $i) &>/dev/null; do
-                if [ "$counter" -ge "$timeout" ]; then echo "TIMEOUT"; exit -1; else (( counter+=5 )); fi
-                echo -n "." >&2
-                sleep 5
-            done
-            echo "FOUND POD!" >&2
-        done
-}
-
-check_deployments(){
-    for name in $1; do
-        echo_sub_step "inspecting deployment '${name}'"
-        local dep_stat=$("${KUBECTL}" -n "$2" get deployments/"${name}")
-
-        echo_info "check if is deployed"
-        if $(echo "$dep_stat" | grep -iq 'No resources found'); then
-            echo "is not deployed"
-            exit -1
-        else
-            echo_step_completed
-        fi
-
-        echo_info "check if is ready"
-        IFS='/' read -ra ready_status_parts <<< "$(echo "$dep_stat" | awk ' FNR > 1 {print $2}')"
-        if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
-            echo "is not Ready"
-            exit -1
-        else
-            echo_step_completed
-        fi
-        echo
-    done
-}
-
-check_pods(){
-    pods=$("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods)
-    count=$(echo "$pods" | wc -l)
-    if (("${count}"-1 != "${1}")); then
-        sleep 10
-        "${KUBECTL}" get events -A
-        sleep 20
-        echo_error "unexpected number of pods"
-        exit -1
-    fi
-    echo "$pods"
-    while read -r pod_stat; do
-        name=$(echo "$pod_stat" | awk '{print $1}')
-        echo_sub_step "inspecting pod '${name}'"
-
-        if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Completed'); then
-            echo_info "is not completed, continuing with further checks"
-        else
-            echo_info "is completed, foregoing further checks"
-            echo_step_completed
-            continue
-        fi
-
-        echo_info "check if is ready"
-        IFS='/' read -ra ready_status_parts <<< "$(echo "$pod_stat" | awk '{print $2}')"
-        if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
-            echo_error "is not ready"
-            exit -1
-        else
-            echo_step_completed
-        fi
-
-        echo_info "check if is running"
-        if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Running'); then
-            echo_error "is not running"
-            exit -1
-        else
-            echo_step_completed
-        fi
-
-        echo_info "check if has restarts"
-        if (( $(echo "$pod_stat" | awk '{print $4}') > 0 )); then
-            echo_error "has restarts"
-            exit -1
-        else
-            echo_step_completed
-        fi
-        echo
-    done <<< "$(echo "$pods" | awk 'FNR>1')"
-}
-
 # ------------------------------
 projectdir="$( cd "$( dirname "${BASH_SOURCE[0]}")"/../.. && pwd )"
 
@@ -144,7 +47,7 @@ CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-controller-${SAFEHOSTARCH}"
 version_tag="$(cat ${projectdir}/_output/version)"
 # tag as latest version to load into kind cluster
 PACKAGE_CONTROLLER_IMAGE="${DOCKER_REGISTRY}/${PROJECT_NAME}-controller:${VERSION}"
-K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-INTTESTS}"
+K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-inttests}"
 
 CROSSPLANE_NAMESPACE="crossplane-system"
 PACKAGE_NAME="provider-helm"
@@ -168,7 +71,8 @@ echo "created cache dir at ${CACHE_PATH}"
 docker save "${BUILD_IMAGE}" -o "${CACHE_PATH}/${PACKAGE_NAME}.xpkg" && chmod 644 "${CACHE_PATH}/${PACKAGE_NAME}.xpkg"
 
 # create kind cluster with extra mounts
-echo_step "creating k8s cluster using kind"
+KIND_NODE_IMAGE="kindest/node:${KIND_NODE_IMAGE_TAG}"
+echo_step "creating k8s cluster using kind ${KIND_VERSION} and node image ${KIND_NODE_IMAGE}"
 KIND_CONFIG="$( cat <<EOF
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
@@ -179,17 +83,12 @@ nodes:
     containerPath: /cache
 EOF
 )"
-echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --config=-
+echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${KIND_NODE_IMAGE}" --config=-
 
 # tag controller image and load it into kind cluster
 docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_CONTROLLER_IMAGE}"
 "${KIND}" load docker-image "${PACKAGE_CONTROLLER_IMAGE}" --name="${K8S_CLUSTER}"
 
-# wait for kind pods
-echo_step "wait for kind pods"
-kindpods=("kube-apiserver-${BUILD_REGISTRY}-inttests-control-plane" "kube-controller-manager-${BUILD_REGISTRY}-inttests-control-plane" "kube-scheduler-${BUILD_REGISTRY}-inttests-control-plane")
-wait_for_pods_in_namespace 120 "kube-system" "${kindpods[@]}"
-
 # files are not synced properly from host to kind node container on Jenkins, so
 # we must manually copy image from host to node
 echo_step "pre-cache package by copying to kind node"
@@ -254,18 +153,6 @@ echo_step "wait until the pods are up and running"
 
 # ----------- integration tests
 echo_step "--- INTEGRATION TESTS ---"
-echo
-echo_step "check for necessary deployment statuses"
-echo
-echo -------- deployments
-"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments
-
-check_deployments "crossplane" "${CROSSPLANE_NAMESPACE}"
-
-echo_step "check for crossplane pods statuses"
-echo
-echo "--- pods ---"
-check_pods 2
 
 # install package
 echo_step "installing ${PROJECT_NAME} into \"${CROSSPLANE_NAMESPACE}\" namespace"
@@ -283,19 +170,13 @@ EOF
 
 echo "${INSTALL_YAML}" | "${KUBECTL}" apply -f -
 
-"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments
-
-# this is to let package manager unpack pods and start the controller.
-sleep 20
-
 # printing the cache dir contents can be useful for troubleshooting failures
 echo_step "check kind node cache dir contents"
 docker exec "${K8S_CLUSTER}-control-plane" ls -la /cache
 
-echo_step "check for package pod statuses"
-echo
-echo "--- pods ---"
-check_pods 3
+echo_step "waiting for provider to be installed"
+
+kubectl wait "provider.pkg.crossplane.io/${PACKAGE_NAME}" --for=condition=healthy --timeout=60s
 
 echo_step "setup provider"
 SA=$("${KUBECTL}" -n crossplane-system get sa -o name | grep provider-helm | sed -e 's|serviceaccount\/|crossplane-system:|g')
@@ -316,17 +197,22 @@ else
 fi
 
 echo_sub_step "check release deployment"
-check_deployments "wordpress-example" "wordpress"
 
 echo_step "uninstalling ${PROJECT_NAME}"
 
 echo "${INSTALL_YAML}" | "${KUBECTL}" delete -f -
 
 # check pods deleted
-sleep 30
-echo_step "check only crossplane pods remain"
-echo "--- pods ---"
-"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods
-check_pods 2
+timeout=60
+current=0
+step=3
+while [[ $(kubectl get providerrevision.pkg.crossplane.io -o name | wc -l) != "0" ]]; do
+  echo "waiting for provider to be deleted for another $step seconds"
+  current=$current+$step
+  if ! [[ $timeout > $current ]]; then
+    echo_error "timeout of ${timeout}s has been reached"
+  fi
+  sleep $step;
+done
 
 echo_success "Integration tests succeeded!"