From 5c73fc0b41b1f354de351e7600a3160895259afa Mon Sep 17 00:00:00 2001
From: Hasan Turken <turkenh@gmail.com>
Date: Wed, 25 Aug 2021 16:10:05 +0300
Subject: [PATCH 1/4] Integration test improvements

Following https://github.com/crossplane/provider-aws/pull/711

Signed-off-by: Hasan Turken <turkenh@gmail.com>
---
 cluster/integration/integration_tests.sh | 144 +++--------------------
 1 file changed, 15 insertions(+), 129 deletions(-)

diff --git a/cluster/integration/integration_tests.sh b/cluster/integration/integration_tests.sh
index 603e0b2..ab76e4e 100755
--- a/cluster/integration/integration_tests.sh
+++ b/cluster/integration/integration_tests.sh
@@ -32,103 +32,6 @@ echo_error(){
     exit 1
 }
 
-# k8s watchers
-
-wait_for_pods_in_namespace(){
-    local timeout=$1
-    shift
-    namespace=$1
-    shift
-    arr=("$@")
-    local counter=0
-    for i in "${arr[@]}";
-        do
-            echo -n "waiting for pod $i in namespace $namespace..." >&2
-            while ! ("${KUBECTL}" -n $namespace get pod $i) &>/dev/null; do
-                if [ "$counter" -ge "$timeout" ]; then echo "TIMEOUT"; exit -1; else (( counter+=5 )); fi
-                echo -n "." >&2
-                sleep 5
-            done
-            echo "FOUND POD!" >&2
-        done
-}
-
-check_deployments(){
-    for name in $1; do
-        echo_sub_step "inspecting deployment '${name}'"
-        local dep_stat=$("${KUBECTL}" -n "$2" get deployments/"${name}")
-
-        echo_info "check if is deployed"
-        if $(echo "$dep_stat" | grep -iq 'No resources found'); then
-            echo "is not deployed"
-            exit -1
-        else
-            echo_step_completed
-        fi
-
-        echo_info "check if is ready"
-        IFS='/' read -ra ready_status_parts <<< "$(echo "$dep_stat" | awk ' FNR > 1 {print $2}')"
-        if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
-            echo "is not Ready"
-            exit -1
-        else
-            echo_step_completed
-        fi
-        echo
-    done
-}
-
-check_pods(){
-    pods=$("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods)
-    count=$(echo "$pods" | wc -l)
-    if (("${count}"-1 != "${1}")); then
-        sleep 10
-        "${KUBECTL}" get events -A
-        sleep 20
-        echo_error "unexpected number of pods"
-        exit -1
-    fi
-    echo "$pods"
-    while read -r pod_stat; do
-        name=$(echo "$pod_stat" | awk '{print $1}')
-        echo_sub_step "inspecting pod '${name}'"
-
-        if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Completed'); then
-            echo_info "is not completed, continuing with further checks"
-        else
-            echo_info "is completed, foregoing further checks"
-            echo_step_completed
-            continue
-        fi
-
-        echo_info "check if is ready"
-        IFS='/' read -ra ready_status_parts <<< "$(echo "$pod_stat" | awk '{print $2}')"
-        if (("${ready_status_parts[0]}" < "${ready_status_parts[1]}")); then
-            echo_error "is not ready"
-            exit -1
-        else
-            echo_step_completed
-        fi
-
-        echo_info "check if is running"
-        if $(echo "$pod_stat" | awk '{print $3}' | grep -ivq 'Running'); then
-            echo_error "is not running"
-            exit -1
-        else
-            echo_step_completed
-        fi
-
-        echo_info "check if has restarts"
-        if (( $(echo "$pod_stat" | awk '{print $4}') > 0 )); then
-            echo_error "has restarts"
-            exit -1
-        else
-            echo_step_completed
-        fi
-        echo
-    done <<< "$(echo "$pods" | awk 'FNR>1')"
-}
-
 # ------------------------------
 projectdir="$( cd "$( dirname "${BASH_SOURCE[0]}")"/../.. && pwd )"
 
@@ -179,17 +82,12 @@ nodes:
     containerPath: /cache
 EOF
 )"
-echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --config=-
+echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --config=-
 
 # tag controller image and load it into kind cluster
 docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_CONTROLLER_IMAGE}"
 "${KIND}" load docker-image "${PACKAGE_CONTROLLER_IMAGE}" --name="${K8S_CLUSTER}"
 
-# wait for kind pods
-echo_step "wait for kind pods"
-kindpods=("kube-apiserver-${BUILD_REGISTRY}-inttests-control-plane" "kube-controller-manager-${BUILD_REGISTRY}-inttests-control-plane" "kube-scheduler-${BUILD_REGISTRY}-inttests-control-plane")
-wait_for_pods_in_namespace 120 "kube-system" "${kindpods[@]}"
-
 # files are not synced properly from host to kind node container on Jenkins, so
 # we must manually copy image from host to node
 echo_step "pre-cache package by copying to kind node"
@@ -254,18 +152,6 @@ echo_step "wait until the pods are up and running"
 
 # ----------- integration tests
 echo_step "--- INTEGRATION TESTS ---"
-echo
-echo_step "check for necessary deployment statuses"
-echo
-echo -------- deployments
-"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments
-
-check_deployments "crossplane" "${CROSSPLANE_NAMESPACE}"
-
-echo_step "check for crossplane pods statuses"
-echo
-echo "--- pods ---"
-check_pods 2
 
 # install package
 echo_step "installing ${PROJECT_NAME} into \"${CROSSPLANE_NAMESPACE}\" namespace"
@@ -283,19 +169,13 @@ EOF
 
 echo "${INSTALL_YAML}" | "${KUBECTL}" apply -f -
 
-"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get deployments
-
-# this is to let package manager unpack pods and start the controller.
-sleep 20
-
 # printing the cache dir contents can be useful for troubleshooting failures
 echo_step "check kind node cache dir contents"
 docker exec "${K8S_CLUSTER}-control-plane" ls -la /cache
 
-echo_step "check for package pod statuses"
-echo
-echo "--- pods ---"
-check_pods 3
+echo_step "waiting for provider to be installed"
+
+kubectl wait "provider.pkg.crossplane.io/${PACKAGE_NAME}" --for=condition=healthy --timeout=60s
 
 echo_step "setup provider"
 SA=$("${KUBECTL}" -n crossplane-system get sa -o name | grep provider-helm | sed -e 's|serviceaccount\/|crossplane-system:|g')
@@ -323,10 +203,16 @@ echo_step "uninstalling ${PROJECT_NAME}"
 echo "${INSTALL_YAML}" | "${KUBECTL}" delete -f -
 
 # check pods deleted
-sleep 30
-echo_step "check only crossplane pods remain"
-echo "--- pods ---"
-"${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get pods
-check_pods 2
+timeout=60
+current=0
+step=3
+while [[ $(kubectl get providerrevision.pkg.crossplane.io -o name | wc -l) != "0" ]]; do
+  echo "waiting for provider to be deleted for another $step seconds"
+  current=$current+$step
+  if ! [[ $timeout > $current ]]; then
+    echo_error "timeout of ${timeout}s has been reached"
+  fi
+  sleep $step;
+done
 
 echo_success "Integration tests succeeded!"

From 81f47bca156bc6c95aa1b1b5c02c2e1fe40a8d56 Mon Sep 17 00:00:00 2001
From: Hasan Turken <turkenh@gmail.com>
Date: Wed, 25 Aug 2021 16:12:36 +0300
Subject: [PATCH 2/4] Fix integration test kind issue

Details: https://github.com/crossplane/provider-aws/pull/782#issuecomment-888612043

Signed-off-by: Hasan Turken <turkenh@gmail.com>
---
 Makefile                                 | 5 +++--
 cluster/integration/integration_tests.sh | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 3d9ad56..fb77938 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,8 @@ GO111MODULE = on
 
 # ====================================================================================
 # Setup Kubernetes tools
-KIND_VERSION = v0.7.0
+KIND_VERSION ?= v0.11.1
+KIND_NODE_IMAGE_TAG ?= v1.19.11
 USE_HELM3 = true
 -include build/makelib/k8s_tools.mk
 
@@ -81,7 +82,7 @@ e2e.run: test-integration
 # Run integration tests.
 test-integration: $(KIND) $(KUBECTL) $(HELM3)
 	@$(INFO) running integration tests using kind $(KIND_VERSION)
-	@$(ROOT_DIR)/cluster/integration/integration_tests.sh || $(FAIL)
+	@KIND_NODE_IMAGE_TAG=${KIND_NODE_IMAGE_TAG} $(ROOT_DIR)/cluster/integration/integration_tests.sh || $(FAIL)
 	@$(OK) integration tests passed
 
 # Update the submodules, such as the common build scripts.
diff --git a/cluster/integration/integration_tests.sh b/cluster/integration/integration_tests.sh
index ab76e4e..5b05c99 100755
--- a/cluster/integration/integration_tests.sh
+++ b/cluster/integration/integration_tests.sh
@@ -71,7 +71,8 @@ echo "created cache dir at ${CACHE_PATH}"
 docker save "${BUILD_IMAGE}" -o "${CACHE_PATH}/${PACKAGE_NAME}.xpkg" && chmod 644 "${CACHE_PATH}/${PACKAGE_NAME}.xpkg"
 
 # create kind cluster with extra mounts
-echo_step "creating k8s cluster using kind"
+KIND_NODE_IMAGE="kindest/node:${KIND_NODE_IMAGE_TAG}"
+echo_step "creating k8s cluster using kind ${KIND_VERSION} and node image ${KIND_NODE_IMAGE}"
 KIND_CONFIG="$( cat <<EOF
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
@@ -82,7 +83,7 @@ nodes:
     containerPath: /cache
 EOF
 )"
-echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --config=-
+echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${KIND_NODE_IMAGE}" --config=-
 
 # tag controller image and load it into kind cluster
 docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_CONTROLLER_IMAGE}"

From b0ef56152b8af8c92dcf61fade313bda04eb2785 Mon Sep 17 00:00:00 2001
From: Hasan Turken <turkenh@gmail.com>
Date: Wed, 25 Aug 2021 16:29:52 +0300
Subject: [PATCH 3/4] Update build submodule to latest

Signed-off-by: Hasan Turken <turkenh@gmail.com>
---
 build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build b/build
index a6cd5ea..b9e5c80 160000
--- a/build
+++ b/build
@@ -1 +1 @@
-Subproject commit a6cd5ea19f2b85e724656eb0f1e6a2f810069c1a
+Subproject commit b9e5c80250983700f027b27505046f53f3d219f6

From 1b5e3c8adbea9958cc54d284dcfd9425d24bb15d Mon Sep 17 00:00:00 2001
From: Hasan Turken <turkenh@gmail.com>
Date: Mon, 6 Sep 2021 22:28:14 +0300
Subject: [PATCH 4/4] Update integration test cluster name to match kind regex

Signed-off-by: Hasan Turken <turkenh@gmail.com>
---
 cluster/integration/integration_tests.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cluster/integration/integration_tests.sh b/cluster/integration/integration_tests.sh
index 5b05c99..2613d35 100755
--- a/cluster/integration/integration_tests.sh
+++ b/cluster/integration/integration_tests.sh
@@ -47,7 +47,7 @@ CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-controller-${SAFEHOSTARCH}"
 version_tag="$(cat ${projectdir}/_output/version)"
 # tag as latest version to load into kind cluster
 PACKAGE_CONTROLLER_IMAGE="${DOCKER_REGISTRY}/${PROJECT_NAME}-controller:${VERSION}"
-K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-INTTESTS}"
+K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-inttests}"
 
 CROSSPLANE_NAMESPACE="crossplane-system"
 PACKAGE_NAME="provider-helm"
@@ -197,7 +197,6 @@ else
 fi
 
 echo_sub_step "check release deployment"
-check_deployments "wordpress-example" "wordpress"
 
 echo_step "uninstalling ${PROJECT_NAME}"