From 9b1839938b318eebbf8366ab196f3689f0fe755c Mon Sep 17 00:00:00 2001
From: Matthew Wong <mattwon@amazon.com>
Date: Mon, 26 Apr 2021 12:49:34 -0700
Subject: [PATCH] Add eksctl support to e2e scripts

---
 hack/e2e/ebs.sh    |   6 ++-
 hack/e2e/eksctl.sh |  65 +++++++++++++++++++++++++++++
 hack/e2e/kops.sh   |  48 ++++++++++-----------
 hack/e2e/run.sh    | 101 +++++++++++++++++++++++++++++++--------------
 hack/e2e/util.sh   |  10 +++++
 5 files changed, 171 insertions(+), 59 deletions(-)
 create mode 100644 hack/e2e/eksctl.sh

diff --git a/hack/e2e/ebs.sh b/hack/e2e/ebs.sh
index 0ca3646bc1..9a80611d20 100644
--- a/hack/e2e/ebs.sh
+++ b/hack/e2e/ebs.sh
@@ -6,11 +6,13 @@ BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 source "${BASE_DIR}"/util.sh
 
 function ebs_check_migration() {
+  KUBECONFIG=${1}
+
   loudecho "Checking migration"
   # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
   # Find the controller-manager log and read its metrics to verify
-  NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
-  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &
+  NODE=$(kubectl get node -l kubernetes.io/role=master -o json --kubeconfig "${KUBECONFIG}" | jq -r ".items[].metadata.name")
+  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system --kubeconfig "${KUBECONFIG}" &
 
   # Ensure port forwarding succeeded
   n=0
diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl.sh
new file mode 100644
index 0000000000..ed244fb5ee
--- /dev/null
+++ b/hack/e2e/eksctl.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -uo pipefail
+
+function eksctl_install() {
+  INSTALL_PATH=${1}
+  if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then
+    EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz"
+    curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
+    chmod +x "${INSTALL_PATH}"/eksctl
+  fi
+}
+
+function eksctl_create_cluster() {
+  SSH_KEY_PATH=${1}
+  CLUSTER_NAME=${2}
+  BIN=${3}
+  ZONES=${4}
+  INSTANCE_TYPE=${5}
+  K8S_VERSION=${6}
+  CLUSTER_FILE=${7}
+  KUBECONFIG=${8}
+
+  generate_ssh_key "${SSH_KEY_PATH}"
+
+  CLUSTER_NAME="${CLUSTER_NAME//./-}"
+
+  set +e
+  if ${BIN} get cluster "${CLUSTER_NAME}"; then
+    set -e
+    loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${BIN} upgrade cluster -f "${CLUSTER_FILE}"
+  else
+    set -e
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
+    ${BIN} create cluster \
+      --managed \
+      --ssh-access \
+      --ssh-public-key "${SSH_KEY_PATH}".pub \
+      --zones "${ZONES}" \
+      --nodes=3 \
+      --instance-types="${INSTANCE_TYPE}" \
+      --version="${K8S_VERSION}" \
+      --dry-run \
+      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+
+    # TODO implement patching
+
+    loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
+    ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
+  fi
+
+  loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"
+
+  loudecho "Getting cluster ${CLUSTER_NAME}"
+  ${BIN} get cluster "${CLUSTER_NAME}"
+  return $?
+}
+
+function eksctl_delete_cluster() {
+  BIN=${1}
+  CLUSTER_NAME=${2}
+  loudecho "Deleting cluster ${CLUSTER_NAME}"
+  ${BIN} delete cluster "${CLUSTER_NAME}"
+}
diff --git a/hack/e2e/kops.sh b/hack/e2e/kops.sh
index c023366767..91988db642 100644
--- a/hack/e2e/kops.sh
+++ b/hack/e2e/kops.sh
@@ -19,33 +19,28 @@ function kops_install() {
 
 function kops_create_cluster() {
   SSH_KEY_PATH=${1}
-  KOPS_STATE_FILE=${2}
-  CLUSTER_NAME=${3}
-  KOPS_BIN=${4}
-  ZONES=${5}
-  INSTANCE_TYPE=${6}
-  K8S_VERSION=${7}
-  TEST_DIR=${8}
-  KOPS_PATCH_FILE=${10}
-
-  if [[ ! -e ${SSH_KEY_PATH} ]]; then
-    loudecho "Generating SSH key $SSH_KEY_PATH"
-    ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
-  else
-    loudecho "Reusing SSH key $SSH_KEY_PATH"
-  fi
+  CLUSTER_NAME=${2}
+  BIN=${3}
+  ZONES=${4}
+  INSTANCE_TYPE=${5}
+  K8S_VERSION=${6}
+  CLUSTER_FILE=${7}
+  KUBECONFIG=${8}
+  KOPS_PATCH_FILE=${9}
+  KOPS_STATE_FILE=${10}
 
-  CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.json
+  generate_ssh_key "${SSH_KEY_PATH}"
 
   set +e
-  if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
+  if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
     set -e
     loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   else
     set -e
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
-    ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
+    ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
+      --ssh-public-key="${SSH_KEY_PATH}".pub \
       --zones "${ZONES}" \
       --node-count=3 \
       --node-size="${INSTANCE_TYPE}" \
@@ -57,26 +52,27 @@ function kops_create_cluster() {
     kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE"
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    kops create secret --state "${KOPS_STATE_FILE}" --name "${CLUSTER_NAME}" sshpublickey admin -i "${SSH_KEY_PATH}".pub
   fi
 
   loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" \
-    --ssh-public-key="${SSH_KEY_PATH}".pub --yes
+  ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
 
-  ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin
+  loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}"
+  ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
 
   loudecho "Validating cluster ${CLUSTER_NAME}"
-  ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
+  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
   return $?
 }
 
 function kops_delete_cluster() {
-  KOPS_BIN=${1}
+  BIN=${1}
   CLUSTER_NAME=${2}
   KOPS_STATE_FILE=${3}
   loudecho "Deleting cluster ${CLUSTER_NAME}"
-  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+  ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
 
 # TODO switch this to python, all this hacking with jq stinks!
diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh
index 8f883ad6cc..3eeac41142 100755
--- a/hack/e2e/run.sh
+++ b/hack/e2e/run.sh
@@ -19,6 +19,7 @@ set -euo pipefail
 BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 source "${BASE_DIR}"/ebs.sh
 source "${BASE_DIR}"/ecr.sh
+source "${BASE_DIR}"/eksctl.sh
 source "${BASE_DIR}"/helm.sh
 source "${BASE_DIR}"/kops.sh
 source "${BASE_DIR}"/util.sh
@@ -29,10 +30,13 @@ DRIVER_START_TIME_THRESHOLD_SECONDS=60
 
 TEST_ID=${TEST_ID:-$RANDOM}
 CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local
+CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
 
 TEST_DIR=${BASE_DIR}/csi-test-artifacts
 BIN_DIR=${TEST_DIR}/bin
 SSH_KEY_PATH=${TEST_DIR}/id_rsa
+CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.json
+KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"}
 
 REGION=${AWS_REGION:-us-west-2}
 ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
@@ -43,15 +47,17 @@ AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
 IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}}
 IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}}
 
-K8S_VERSION=${K8S_VERSION:-1.20.4}
-KOPS_VERSION=${KOPS_VERSION:-1.20.0-beta.2}
+# kops: must include patch version (e.g. 1.19.1)
+# eksctl: mustn't include patch version (e.g. 1.19)
+K8S_VERSION=${K8S_VERSION:-1.20.6}
+
+KOPS_VERSION=${KOPS_VERSION:-1.20.0}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
 
 HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml}
 
 TEST_PATH=${TEST_PATH:-"./tests/e2e/..."}
-KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"}
 ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"}
 GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"}
 GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
@@ -67,9 +73,18 @@ loudecho "Testing in region ${REGION} and zones ${ZONES}"
 mkdir -p "${BIN_DIR}"
 export PATH=${PATH}:${BIN_DIR}
 
-loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}"
-kops_install "${BIN_DIR}" "${KOPS_VERSION}"
-KOPS_BIN=${BIN_DIR}/kops
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+  loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}"
+  kops_install "${BIN_DIR}" "${KOPS_VERSION}"
+  KOPS_BIN=${BIN_DIR}/kops
+elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+  loudecho "Installing eksctl latest to ${BIN_DIR}"
+  eksctl_install "${BIN_DIR}"
+  EKSCTL_BIN=${BIN_DIR}/eksctl
+else
+  loudecho "${CLUSTER_TYPE} must be kops or eksctl!"
+  exit 1
+fi
 
 loudecho "Installing helm to ${BIN_DIR}"
 helm_install "${BIN_DIR}"
@@ -88,19 +103,34 @@ ecr_build_and_push "${REGION}" \
   "${IMAGE_NAME}" \
   "${IMAGE_TAG}"
 
-kops_create_cluster \
-  "$SSH_KEY_PATH" \
-  "$KOPS_STATE_FILE" \
-  "$CLUSTER_NAME" \
-  "$KOPS_BIN" \
-  "$ZONES" \
-  "$INSTANCE_TYPE" \
-  "$K8S_VERSION" \
-  "$TEST_DIR" \
-  "$BASE_DIR" \
-  "$KOPS_PATCH_FILE"
-if [[ $? -ne 0 ]]; then
-  exit 1
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+  kops_create_cluster \
+    "$SSH_KEY_PATH" \
+    "$CLUSTER_NAME" \
+    "$KOPS_BIN" \
+    "$ZONES" \
+    "$INSTANCE_TYPE" \
+    "$K8S_VERSION" \
+    "$CLUSTER_FILE" \
+    "$KUBECONFIG" \
+    "$KOPS_PATCH_FILE" \
+    "$KOPS_STATE_FILE"
+  if [[ $? -ne 0 ]]; then
+    exit 1
+  fi
+elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+  eksctl_create_cluster \
+    "$SSH_KEY_PATH" \
+    "$CLUSTER_NAME" \
+    "$EKSCTL_BIN" \
+    "$ZONES" \
+    "$INSTANCE_TYPE" \
+    "$K8S_VERSION" \
+    "$CLUSTER_FILE" \
+    "$KUBECONFIG"
+  if [[ $? -ne 0 ]]; then
+    exit 1
+  fi
 fi
 
 loudecho "Deploying driver"
@@ -111,15 +141,17 @@ startSec=$(date +'%s')
   --set image.tag="${IMAGE_TAG}" \
   -f "${HELM_VALUES_FILE}" \
   --wait \
+  --kubeconfig "${KUBECONFIG}" \
   ./charts/"${DRIVER_NAME}"
 
 if [[ -r "${EBS_SNAPSHOT_CRD}" ]]; then
   loudecho "Deploying snapshot CRD"
-  kubectl apply -f "$EBS_SNAPSHOT_CRD"
+  kubectl apply -f "$EBS_SNAPSHOT_CRD" \
+    --kubeconfig "${KUBECONFIG}"
   # TODO deploy snapshot controller too instead of including in helm chart
 fi
 endSec=$(date +'%s')
-secondUsed=$(( (endSec-startSec)/1 ))
+secondUsed=$(((endSec - startSec) / 1))
 # Set timeout threshold as 20 seconds for now, usually it takes less than 10s to startup
 if [ $secondUsed -gt $DRIVER_START_TIME_THRESHOLD_SECONDS ]; then
   loudecho "Driver start timeout, took $secondUsed but the threshold is $DRIVER_START_TIME_THRESHOLD_SECONDS. Fail the test."
@@ -127,7 +159,6 @@ if [ $secondUsed -gt $DRIVER_START_TIME_THRESHOLD_SECONDS ]; then
 fi
 loudecho "Driver deployment complete, time used: $secondUsed seconds"
 
-
 loudecho "Testing focus ${GINKGO_FOCUS}"
 eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS"
 set -x
@@ -141,7 +172,7 @@ loudecho "TEST_PASSED: ${TEST_PASSED}"
 OVERALL_TEST_PASSED="${TEST_PASSED}"
 if [[ "${EBS_CHECK_MIGRATION}" == true ]]; then
   exec 5>&1
-  OUTPUT=$(ebs_check_migration | tee /dev/fd/5)
+  OUTPUT=$(ebs_check_migration "${KUBECONFIG}" | tee /dev/fd/5)
   MIGRATION_PASSED=$(echo "${OUTPUT}" | tail -1)
   loudecho "MIGRATION_PASSED: ${MIGRATION_PASSED}"
   if [ "${TEST_PASSED}" == 0 ] && [ "${MIGRATION_PASSED}" == 0 ]; then
@@ -153,12 +184,13 @@ if [[ "${EBS_CHECK_MIGRATION}" == true ]]; then
   fi
 fi
 
-PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json | jq -r .items[].metadata.name)
+PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name)
 
 while IFS= read -r POD; do
   loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs"
   set +e
-  kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}"
+  kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \
+    --kubeconfig "${KUBECONFIG}"
   set -e
 done <<< "${PODS}"
 
@@ -167,12 +199,19 @@ if [[ "${CLEAN}" == true ]]; then
 
   loudecho "Removing driver"
   ${HELM_BIN} del "${DRIVER_NAME}" \
-    --namespace kube-system
-
-  kops_delete_cluster \
-    "${KOPS_BIN}" \
-    "${CLUSTER_NAME}" \
-    "${KOPS_STATE_FILE}"
+    --namespace kube-system \
+    --kubeconfig "${KUBECONFIG}"
+
+  if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+    kops_delete_cluster \
+      "${KOPS_BIN}" \
+      "${CLUSTER_NAME}" \
+      "${KOPS_STATE_FILE}"
+  elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+    eksctl_delete_cluster \
+      "${EKSCTL_BIN}" \
+      "${CLUSTER_NAME}"
+  fi
 else
   loudecho "Not cleaning"
 fi
diff --git a/hack/e2e/util.sh b/hack/e2e/util.sh
index 46bb5b16e0..56f23e3f86 100644
--- a/hack/e2e/util.sh
+++ b/hack/e2e/util.sh
@@ -7,3 +7,13 @@ function loudecho() {
   echo "## ${1}"
   echo "#"
 }
+
+function generate_ssh_key() {
+  SSH_KEY_PATH=${1}
+  if [[ ! -e ${SSH_KEY_PATH} ]]; then
+    loudecho "Generating SSH key $SSH_KEY_PATH"
+    ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
+  else
+    loudecho "Reusing SSH key $SSH_KEY_PATH"
+  fi
+}