run e2e tests on EKS

Signed-off-by: adrienjt <adrienjt@users.noreply.github.com>

.github/workflows/go.yml

@@ -49,9 +49,11 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     continue-on-error: ${{ matrix.experimental }}
+    environment: ${{ (matrix.k8s_distrib == 'eks' && 'EKS') || null }}
     strategy:
       fail-fast: false
       matrix:
+        k8s_distrib: ${{ github.repository_owner == 'admiraltyio' && [ 'kind', 'eks' ] || [ 'kind' ] }}
         k8s_version: [ "1.17", "1.18", "1.19", "1.20", "1.21" ]
         experimental: [ false ]
         # workflow succeeds even if experimental job fails,
@@ -83,8 +85,16 @@ jobs:
       - name: Install kubectl, helm, kind
         run: ./test/e2e/install_dependencies.sh
 
+      - name: Configure AWS credentials
+        if: matrix.k8s_distrib == 'eks'
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
       - name: End-to-end test
-        run: VERSION=$GITHUB_SHA K8S_VERSION=${{ matrix.k8s_version }} ./test/e2e/e2e.sh
+        run: VERSION=$GITHUB_SHA K8S_VERSION=${{ matrix.k8s_version }} K8S_DISTRIB=${{ matrix.k8s_distrib }} ./test/e2e/e2e.sh
 
       - name: Archive cluster dump
         if: ${{ failure() }}

test/e2e/admiralty.sh

@@ -18,28 +18,32 @@
 set -euo pipefail
 
 VERSION="${VERSION:-dev}"
+REGISTRY="${REGISTRY:-}"
 
 source test/e2e/aliases.sh
-source test/e2e/kind.sh
-source test/e2e/webhook_ready.sh
 
 admiralty_setup() {
   i=$1
   VALUES=$2
 
-  kind load docker-image multicluster-scheduler-agent:$VERSION-amd64 --name cluster$i
-  kind load docker-image multicluster-scheduler-scheduler:$VERSION-amd64 --name cluster$i
-  kind load docker-image multicluster-scheduler-remove-finalizers:$VERSION-amd64 --name cluster$i
-  kind load docker-image multicluster-scheduler-restarter:$VERSION-amd64 --name cluster$i
-
   if ! k $i get ns admiralty; then
     k $i create namespace admiralty
   fi
+  img_prefix=""
+  pull_policy=IfNotPresent
+  if [[ "$REGISTRY" != "" ]]; then
+    img_prefix="$REGISTRY/"
+    pull_policy=Always
+  fi
   h $i upgrade --install multicluster-scheduler charts/multicluster-scheduler -n admiralty -f $VALUES \
-    --set controllerManager.image.repository=multicluster-scheduler-agent \
-    --set scheduler.image.repository=multicluster-scheduler-scheduler \
-    --set postDeleteJob.image.repository=multicluster-scheduler-remove-finalizers \
-    --set restarter.image.repository=multicluster-scheduler-restarter \
+    --set controllerManager.image.repository=${img_prefix}multicluster-scheduler-agent \
+    --set scheduler.image.repository=${img_prefix}multicluster-scheduler-scheduler \
+    --set postDeleteJob.image.repository=${img_prefix}multicluster-scheduler-remove-finalizers \
+    --set restarter.image.repository=${img_prefix}multicluster-scheduler-restarter \
+    --set controllerManager.image.pullPolicy=$pull_policy \
+    --set scheduler.image.pullPolicy=$pull_policy \
+    --set postDeleteJob.image.pullPolicy=$pull_policy \
+    --set restarter.image.pullPolicy=$pull_policy \
     --set controllerManager.image.tag=$VERSION-amd64 \
     --set scheduler.image.tag=$VERSION-amd64 \
     --set postDeleteJob.image.tag=$VERSION-amd64 \
@@ -49,6 +53,53 @@ admiralty_setup() {
   k $i label ns default multicluster-scheduler=enabled --overwrite
 }
 
+admiralty_connect() {
+  i=$1
+  j=$2
+
+  if [[ $i == $j ]]; then
+    # if self target
+    cat <<EOF | k $i apply -f -
+kind: Target
+apiVersion: multicluster.admiralty.io/v1alpha1
+metadata:
+  name: c$j
+spec:
+  self: true
+EOF
+  else
+    if k $j cluster-info; then
+      # if cluster j exists
+      cat <<EOF | k $j apply -f -
+kind: Source
+apiVersion: multicluster.admiralty.io/v1alpha1
+metadata:
+  name: cluster$i
+spec:
+  serviceAccountName: cluster$i
+EOF
+      while ! k $j get sa cluster$i; do sleep 1; done
+
+      SECRET_NAME=$(k $j get serviceaccount cluster1 -o json | jq -r .secrets[0].name)
+      TOKEN=$(k $j get secret $SECRET_NAME -o json | jq -r .data.token | base64 --decode)
+      KUBECONFIG=$(k $j config view --minify --raw -o json | jq '.users[0].user={token:"'$TOKEN'"} | .contexts[0].context.namespace="default"')
+      k $i create secret generic c$j --from-literal=config="$KUBECONFIG" --dry-run -o yaml | k $i apply -f -
+    fi
+
+    # if cluster j doesn't exist, this is a misconfigured target
+    # which must be handled gracefully
+    cat <<EOF | k $i apply -f -
+kind: Target
+apiVersion: multicluster.admiralty.io/v1alpha1
+metadata:
+  name: c$j
+spec:
+  kubeconfigSecret:
+    name: c$j
+EOF
+  fi
+}
+
 if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then
   admiralty_setup "${@}"
 fi

test/e2e/argo.sh

@@ -37,9 +37,6 @@ argo_setup_once() {
     curl -Lo argo "https://github.com/argoproj/argo-workflows/releases/download/v$argo_version/argo-$os-$arch"
     chmod +x argo
   fi
-
-  # to speed up container creations (loaded by kind in argo_setup_source and argo_setup_target)
-  docker pull "$argo_img" # may already be on host
 }
 
 argo_setup_source() {
@@ -56,18 +53,12 @@ argo_setup_source() {
   k "$i" delete pod --all -n argo # reload config map
 
   k "$i" apply -f examples/argo-workflows/_service-account.yaml
-
-  # speed up container creations
-  kind load docker-image "$argo_img" --name "cluster$i"
 }
 
 argo_setup_target() {
   i=$1
 
   k "$i" apply -f examples/argo-workflows/_service-account.yaml
-
-  # speed up container creations
-  kind load docker-image "$argo_img" --name "cluster$i"
 }
 
 argo_test() {

test/e2e/cert-manager.sh

@@ -18,7 +18,8 @@
 set -euo pipefail
 
 source test/e2e/aliases.sh
-source test/e2e/webhook_ready.sh
+
+cert_manager_version=v0.16.1
 
 cert_manager_setup_once() {
   helm repo add jetstack https://charts.jetstack.io
@@ -28,12 +29,11 @@ cert_manager_setup_once() {
 cert_manager_setup() {
   i=$1
 
-  k $i apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.16.1/cert-manager.crds.yaml
+  k $i apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/$cert_manager_version/cert-manager.crds.yaml
   if ! k $i get ns cert-manager; then
     k $i create ns cert-manager
   fi
-  h $i upgrade --install cert-manager jetstack/cert-manager -n cert-manager --version v0.16.1 --wait --debug --timeout=1m
-  #  webhook_ready $i cert-manager cert-manager-webhook cert-manager-webhook cert-manager-webhook-tls
+  h $i upgrade --install cert-manager jetstack/cert-manager -n cert-manager --version $cert_manager_version --wait --debug --timeout=1m
 }
 
 if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then

test/e2e/e2e.sh

@@ -21,7 +21,8 @@ source test/e2e/aliases.sh
 source test/e2e/admiralty.sh
 source test/e2e/argo.sh
 source test/e2e/cert-manager.sh
-source test/e2e/kind.sh
+source test/e2e/k8s/kind.sh
+source test/e2e/k8s/eks.sh
 source test/e2e/follow/test.sh
 source test/e2e/logs/test.sh
 source test/e2e/exec/test.sh
@@ -30,32 +31,60 @@ source test/e2e/virtual-node-labels/test.sh
 source test/e2e/webhook_ready.sh
 source test/e2e/no-rogue-finalizer/test.sh
 
+K8S_DISTRIB="${K8S_DISTRIB:-kind}"
+
 argo_setup_once
 cert_manager_setup_once
 
+case "$K8S_DISTRIB" in
+  kind)
+    kind_setup_once
+    create_cluster=kind_setup
+    async=false
+    ;;
+  eks)
+    eks_setup_once
+    create_cluster=eks_setup
+    async=true
+    ;;
+  *)
+    echo "unknown Kubernetes distribution $K8S_DISTRIB" >&2
+    exit 1
+    ;;
+esac
+
 cluster_dump() {
   if [ $? -ne 0 ]; then
-    k 1 cluster-info dump -A --output-directory cluster-dump/1
-    k 2 cluster-info dump -A --output-directory cluster-dump/2
+    for i in 1 2; do
+      k $i cluster-info dump -A --output-directory cluster-dump/$i
+    done
   fi
 }
 trap cluster_dump EXIT
 
+pids=()
+
 for i in 1 2; do
-  kind_setup $i
-  cert_manager_setup $i
-  admiralty_setup $i test/e2e/values.yaml
+  if [ $async = true ]; then
+    $create_cluster $i &
+    pids+=($!)
+  else
+    $create_cluster $i
+  fi
 done
 
-k 2 apply -f test/e2e/topologies/namespaced-burst/cluster2/source.yaml
-while ! k 2 get sa cluster1; do sleep 1; done
+for pid in "${pids[@]}"; do
+  wait $pid
+done
 
-SECRET_NAME=$(k 2 get serviceaccount cluster1 -o json | jq -r .secrets[0].name)
-TOKEN=$(k 2 get secret $SECRET_NAME -o json | jq -r .data.token | base64 --decode)
-KUBECONFIG=$(k 2 config view --minify --raw -o json | jq '.users[0].user={token:"'$TOKEN'"} | .contexts[0].context.namespace="default"')
-k 1 create secret generic c2 --from-literal=config="$KUBECONFIG" --dry-run -o yaml | k 1 apply -f -
+for i in 1 2; do
+  cert_manager_setup $i
+  REGISTRY=$registry admiralty_setup $i test/e2e/values.yaml
+done
 
-k 1 apply -f test/e2e/topologies/namespaced-burst/cluster1/targets.yaml
+for j in 1 2 3; do
+  admiralty_connect 1 $j
+done
 
 argo_setup_source 1
 argo_setup_target 2
@@ -68,8 +97,10 @@ webhook_ready 1 admiralty multicluster-scheduler-controller-manager multicluster
 
 argo_test 1 2
 follow_test 1 2
+#if [[ "$K8S_DISTRIB" != eks || "$K8S_VERSION" == "1.17" || "$K8S_VERSION" == "1.18" ]]; then
 logs_test 1 2
 exec_test 1 2
+#fi
 ingress_test 1 2
 virtual-node-labels_test 1 2
 no-rogue-finalizer_test

test/e2e/install_dependencies.sh

@@ -33,3 +33,6 @@ sudo mv ./kind /usr/local/bin/kind
 curl -LO https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz
 tar -zxvf helm-v3.2.4-linux-amd64.tar.gz
 sudo mv linux-amd64/helm /usr/local/bin/helm
+
+curl --silent --location https://github.com/weaveworks/eksctl/releases/download/v0.67.0/eksctl_Linux_amd64.tar.gz | tar xz -C /tmp
+sudo mv /tmp/eksctl /usr/local/bin

test/e2e/k8s/eks.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Copyright 2021 The Multicluster-Scheduler Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+source test/e2e/aliases.sh
+
+REGION="${REGION:-"us-west-2"}"
+VERSION="${VERSION:-dev}"
+
+eks_setup_once() {
+  aws_account_id="$(aws sts get-caller-identity | jq -r .Account)"
+
+  registry="$aws_account_id.dkr.ecr.$REGION.amazonaws.com"
+  aws ecr get-login-password | docker login --username AWS --password-stdin "$registry"
+  # TODO... use https://github.com/awslabs/amazon-ecr-credential-helper
+
+  imgs=(
+    multicluster-scheduler-agent
+    multicluster-scheduler-remove-finalizers
+    multicluster-scheduler-scheduler
+    multicluster-scheduler-restarter
+  )
+
+  for img in "${imgs[@]}"; do
+    if ! aws ecr describe-repositories --region $REGION --repository-names $img; then
+      aws ecr create-repository --region $REGION --repository-name $img
+    fi
+    ARCHS=amd64 VERSION="${VERSION}" REGISTRY=$registry IMG="$img" ./release/image.sh
+  done
+
+  K8S_VERSION="${K8S_VERSION:-"1.21"}"
+}
+
+eks_setup() {
+  i=$1
+
+  cluster=cluster$i
+  kubeconfig="kubeconfig-$cluster"
+  ekscluster=$cluster-${K8S_VERSION//./-} # for concurrency
+
+  if ! eksctl get cluster --name $ekscluster --region $REGION; then
+    eksctl create cluster --name $ekscluster --region $REGION --managed --kubeconfig $kubeconfig --version $K8S_VERSION
+  fi
+}
+
+if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then
+  eks_setup_once
+  eks_setup "${@}"
+fi

test/e2e/kind.sh → test/e2e/k8s/kind.sh

@@ -33,7 +33,19 @@ kind_images[1.14]="kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27
 # "known to work well" (k8s 1.22 wasn't released when kind 0.11.1 was released)
 kind_images[1.22]="kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047"
 
-K8S_VERSION="${K8S_VERSION:-"1.21"}"
+VERSION="${VERSION:-dev}"
+
+kind_setup_once() {
+  # to speed up container creations (loaded by in kind_setup)
+  # may already be on host
+  docker pull "$argo_img"
+  docker pull quay.io/jetstack/cert-manager-controller:$cert_manager_version
+  docker pull quay.io/jetstack/cert-manager-webhook:$cert_manager_version
+  docker pull quay.io/jetstack/cert-manager-cainjector:$cert_manager_version
+  registry=""
+
+  K8S_VERSION="${K8S_VERSION:-"1.21"}"
+}
 
 kind_setup() {
   i=$1
@@ -47,8 +59,16 @@ kind_setup() {
   kind get kubeconfig --name $CLUSTER --internal | \
     sed "s/${CLUSTER}-control-plane/${NODE_IP}/g" >kubeconfig-$CLUSTER
   k $i apply -f test/e2e/must-run-as-non-root.yaml
+
+  # speed up container creations
+  kind load docker-image multicluster-scheduler-agent:$VERSION-amd64 --name cluster$i
+  kind load docker-image multicluster-scheduler-scheduler:$VERSION-amd64 --name cluster$i
+  kind load docker-image multicluster-scheduler-remove-finalizers:$VERSION-amd64 --name cluster$i
+  kind load docker-image multicluster-scheduler-restarter:$VERSION-amd64 --name cluster$i
+  kind load docker-image "$argo_img" --name "cluster$i"
 }
 
 if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]]; then
+  kind_setup_once
   kind_setup $1
 fi