e2e: parallelize end-to-end tests (#150)

e2e: parallelize end-to-end tests

.circleci/config.yml

@@ -4,6 +4,7 @@ jobs:
     working_directory: /home/circleci/go/src/github.com/fluxcd/helm-operator
     machine:
       image: ubuntu-1604:201903-01
+    resource_class: large
     environment:
       GO_VERSION: 1.13.3
       # We don't need a GOPATH but CircleCI defines it, so we override it
@@ -52,7 +53,16 @@ jobs:
       - run: make test TEST_FLAGS="-race -timeout 60s"
       - run: make all
       - run: make test-docs
-      - run: make e2e
+      - run:
+          name: End-to-end Helm v2
+          command: E2E_KIND_CLUSTER_NUM=4 make e2e
+          environment:
+            HELM_VERSION: v2
+      - run:
+          name: End-to-end Helm v3
+          command: E2E_KIND_CLUSTER_NUM=4 make e2e
+          environment:
+            HELM_VERSION: v3
       - save_cache:
           key: cache-{{ checksum "Makefile" }}
           paths:

Makefile

@@ -5,7 +5,7 @@ SUDO := $(shell docker info > /dev/null 2> /dev/null || echo "sudo")
 
 TEST_FLAGS?=
 
-BATS_VERSION := 1.1.0
+BATS_COMMIT := 3a1c2f28be260f8687ff83183cef4963faabedd6
 SHELLCHECK_VERSION := 0.7.0
 SHFMT_VERSION := 2.6.4
 
@@ -126,12 +126,13 @@ cache/%/shfmt-$(SHFMT_VERSION):
 	mkdir -p cache/$*
 	curl --fail -L -o $@ "https://github.com/mvdan/sh/releases/download/v$(SHFMT_VERSION)/shfmt_v$(SHFMT_VERSION)_`echo $* | tr - _`"
 
-test/e2e/bats: cache/bats-v$(BATS_VERSION).tar.gz
+test/e2e/bats: cache/bats-core-$(BATS_COMMIT).tar.gz
 	mkdir -p $@
 	tar -C $@ --strip-components 1 -xzf $<
 
-cache/bats-v$(BATS_VERSION).tar.gz:
-	curl --fail -L -o $@ https://github.com/bats-core/bats-core/archive/v$(BATS_VERSION).tar.gz
+cache/bats-core-$(BATS_COMMIT).tar.gz:
+	# Use 2opremio's fork until https://github.com/bats-core/bats-core/pull/255 is merged
+	curl --fail -L -o $@ https://github.com/2opremio/bats-core/archive/$(BATS_COMMIT).tar.gz
 
 generate-deploy: pkg/install/generated_templates.gogen.go
 	cd deploy && go run ../pkg/install/generate.go deploy

test/e2e/10_helm_chart.bats

@@ -1,10 +1,10 @@
 #!/usr/bin/env bats
 
-load lib/env
-load lib/install
-load lib/poll
-
 function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+
   kubectl create namespace "$E2E_NAMESPACE"
   install_git_srv
   install_tiller

test/e2e/lib/env.bash

@@ -11,3 +11,8 @@ export KNOWN_HOSTS
 GITCONFIG=$(cat "${FIXTURES_DIR}/gitconfig")
 export GITCONFIG
 export HELM_VERSION=${HELM_VERSION}
+
+# Wire the test to the right cluster when tests are run in parallel
+if eval [ -n '$KUBECONFIG_SLOT_'"${BATS_JOB_SLOT}" ]; then
+  eval export KUBECONFIG='$KUBECONFIG_SLOT_'"${BATS_JOB_SLOT}"
+fi

test/e2e/run.bash

@@ -10,36 +10,45 @@ ROOT_DIR="$(git rev-parse --show-toplevel)"
 E2E_DIR="${ROOT_DIR}/test/e2e"
 CACHE_DIR="${ROOT_DIR}/cache/$CURRENT_OS_ARCH"
 
-KIND_VERSION="v0.5.1"
+KIND_VERSION="v0.6.1"
 KIND_CACHE_PATH="${CACHE_DIR}/kind-$KIND_VERSION"
-KIND_CLUSTER=helm-operator-e2e
-USING_KIND=false
+KIND_CLUSTER_PREFIX=helm-operator-e2e
 
 # shellcheck disable=SC1090
 source "${E2E_DIR}/lib/defer.bash"
 trap run_deferred EXIT
 
-# Check if there is a kubernetes cluster running, otherwise use Kind
-if ! kubectl version > /dev/null 2>&1; then
+function install_kind() {
   if [ ! -f "${KIND_CACHE_PATH}" ]; then
     echo '>>> Downloading Kind'
     mkdir -p "${CACHE_DIR}"
     curl -sL "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${CURRENT_OS_ARCH}" -o "${KIND_CACHE_PATH}"
   fi
-  echo '>>> Creating Kind Kubernetes cluster'
   cp "${KIND_CACHE_PATH}" "${ROOT_DIR}/test/bin/kind"
   chmod +x "${ROOT_DIR}/test/bin/kind"
-  kind create cluster --name "${KIND_CLUSTER}" --wait 5m
-  defer kind --name "${KIND_CLUSTER}" delete cluster > /dev/null 2>&1 || true
-  KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)"
-  export KUBECONFIG
-  USING_KIND=true
-  kubectl get pods --all-namespaces
-fi
+}
+
+# Create multiple Kind clusters and run jobs in parallel?
+# Let users specify how many, e.g. with E2E_KIND_CLUSTER_NUM=3 make e2e
+E2E_KIND_CLUSTER_NUM=${E2E_KIND_CLUSTER_NUM:-1}
 
-if [ "${USING_KIND}" = 'true' ]; then
-  echo '>>> Loading images into the Kind cluster'
-  kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/fluxcd/helm-operator:latest'
+# Check if there is a kubernetes cluster running, otherwise use Kind
+if ! kubectl version > /dev/null 2>&1; then
+  install_kind
+
+  echo '>>> Creating Kind Kubernetes cluster(s)'
+  seq 1 "${E2E_KIND_CLUSTER_NUM}" | time parallel -- kind create cluster --name "${KIND_CLUSTER_PREFIX}-{}" --wait 5m
+  for I in $(seq 1 "${E2E_KIND_CLUSTER_NUM}"); do
+    defer kind --name "${KIND_CLUSTER_PREFIX}-${I}" delete cluster > /dev/null 2>&1 || true
+    # Wire tests with the right cluster based on their BATS_JOB_SLOT env variable
+    eval export KUBECONFIG_SLOT_"${I}"="$(kind --name="${KIND_CLUSTER_PREFIX}-${I}" get kubeconfig-path)"
+  done
+
+  echo '>>> Loading images into the Kind cluster(s)'
+  seq 1 "${E2E_KIND_CLUSTER_NUM}" | time parallel -- kind --name "${KIND_CLUSTER_PREFIX}-{}" load docker-image 'docker.io/fluxcd/helm-operator:latest'
+  if [ "${E2E_KIND_CLUSTER_NUM}" -gt 1 ]; then
+    BATS_EXTRA_ARGS="--jobs ${E2E_KIND_CLUSTER_NUM}"
+  fi
 fi
 
 echo '>>> Running the tests'
@@ -50,5 +59,5 @@ HELM_VERSION=${HELM_VERSION:-}
   cd "${E2E_DIR}"
   export HELM_VERSION=${HELM_VERSION}
   # shellcheck disable=SC2086
-  "${E2E_DIR}/bats/bin/bats" -t ${E2E_TESTS}
+  "${E2E_DIR}/bats/bin/bats" --tap -t ${BATS_EXTRA_ARGS} ${E2E_TESTS}
 )