👷 Add high-level E2E test steps for Skipper

Add e2e-skipper* files for test setup

It does the following things:
* install Skipper ingress with Kustomize
* load Flagger image onto the local cluster
* install Flagger and Prometheus in the flagger-system namespace

kustomize/skipper/kustomization.yaml

@@ -0,0 +1,67 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base/prometheus/
+  - ../base/flagger/
+  - https://raw.githubusercontent.com/zalando/skipper/master/docs/kubernetes/deploy/deployment/rbac.yaml
+  - https://raw.githubusercontent.com/zalando/skipper/master/docs/kubernetes/deploy/deployment/service.yaml
+  - https://raw.githubusercontent.com/zalando/skipper/master/docs/kubernetes/deploy/deployment/deployment.yaml
+  - namespace.yaml
+patchesStrategicMerge:
+  - patch.yaml
+patches:
+  - target:
+      kind: Deployment
+      name: skipper-ingress
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: skipper-ingress
+      spec:
+        template:
+          metadata:
+            annotations:
+              prometheus.io/path: /metrics
+              prometheus.io/port: "9911"
+              prometheus.io/scrape: "true"
+          spec:
+            nodeSelector:
+              $patch: delete
+            affinity:
+              $patch: delete
+            containers:
+              - name: skipper-ingress
+                image: registry.opensource.zalan.do/pathfinder/skipper:latest
+                ports:
+                  - name: metrics-port
+                    containerPort: 9911
+                resources:
+                  $patch: delete
+                readinessProbe:
+                  initialDelaySeconds: 5
+                args:
+                  - skipper
+                  - -default-filters-prepend=enableAccessLog(4,5)
+                  - -address=:9999
+                  - -disable-metrics-compat
+                  - -enable-connection-metrics
+                  - -enable-profile
+                  - -enable-ratelimits
+                  - -experimental-upgrade
+                  - -histogram-metric-buckets=.01,1,10,100
+                  - -kubernetes-in-cluster
+                  - -kubernetes-path-mode=path-prefix
+                  - -kubernetes
+                  - -lb-healthcheck-interval=3s
+                  - -max-audit-body=0
+                  - -max-idle-connection-backend=0
+                  - -metrics-exp-decay-sample
+                  - -metrics-flavour=prometheus
+                  - -proxy-preserve-host
+                  - -route-backend-metrics
+                  - -route-backend-error-counters
+                  - -route-response-metrics
+                  - -serve-host-metrics
+                  - -serve-route-metrics
+                  - -whitelisted-healthcheck-cidr=0.0.0.0/0 # kind uses other IP addresse

kustomize/skipper/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: flagger-system

kustomize/skipper/patch.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -log-level=debug
+            - -mesh-provider=kubernetes
+            - -metrics-server=http://flagger-prometheus:9090

test/e2e-skipper-canary.yaml

@@ -0,0 +1,66 @@
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: skipper
+  progressDeadlineSeconds: 120
+  revertOnDeletion: true
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  ingressRef:
+    apiVersion: networking.k8s.io/v1beta1
+    kind: Ingress
+    name: podinfo-ingress
+  service:
+    # service name (defaults to targetRef.name)
+    name: podinfo-service
+    # ClusterIP port number
+    port: 80
+    # container port name or number (optional)
+    targetPort: http
+    # port name can be http or grpc (default http)
+    # portName: http
+    # add all the other container ports
+    # to the ClusterIP services (default false)
+    # portDiscovery: false
+  analysis:
+    interval: 15s
+    threshold: 5
+    maxWeight: 100
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      interval: 15s
+      # minimum req success rate (non 5xx responses)
+      # percentage (0-100)
+      thresholdRange:
+        min: 99
+    - name: request-duration
+      interval: 15s
+      # maximum req duration P99
+      # milliseconds
+      thresholdRange:
+        max: 500
+    webhooks:
+      - name: gate
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 10s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-service-canary/token | grep token"
+      - name: "load test"
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://skipper-ingress.kube-system"
+          logCmdOutput: "true"

test/e2e-skipper-cleanup.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Deleting Skipper Ingress'
+kustomize build test/skipper/ | kubectl delete -f -
+
+echo '>>> Deleting Flagger'
+kubectl delete -k ${REPO_ROOT}/kustomize/kubernetes
+
+echo '>>> Cleanup test namespace'
+kubectl delete namespace test --ignore-not-found=true

test/e2e-skipper-tests.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind and Skipper ingress controller
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Creating test namespace'
+kubectl create namespace test || true
+
+echo '>>> Initialising workload'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload-ingress.yaml
+
+echo '>>> Installing load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Create canary CRD'
+kubectl apply -f ${REPO_ROOT}/test/e2e-skipper-canary.yaml
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+  kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+  sleep 5
+  count=$(($count + 1))
+  if [[ ${count} -eq ${retries} ]]; then
+    kubectl -n flagger-system logs deployment/flagger
+    echo "No more retries left"
+    exit 1
+  fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+failed=false
+until ${ok}; do
+  kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false
+  if ${failed}; then
+    kubectl -n flagger-system logs deployment/flagger
+    echo "Canary failed!"
+    exit 1
+  fi
+  kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+  sleep 10
+  kubectl -n flagger-system logs deployment/flagger --tail 1
+  count=$(($count + 1))
+  if [[ ${count} -eq ${retries} ]]; then
+    kubectl -n test describe deployment/podinfo
+    kubectl -n test describe deployment/podinfo-primary
+    kubectl -n flagger-system logs deployment/flagger
+    echo "No more retries left"
+    exit 1
+  fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+  kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+  sleep 5
+  count=$(($count + 1))
+  if [[ ${count} -eq ${retries} ]]; then
+    kubectl -n flagger-system logs deployment/flagger
+    echo "No more retries left"
+    exit 1
+  fi
+done
+
+echo '✔ Canary promotion test passed'

test/e2e-skipper.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Loading Flagger image'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Skipper Ingress, Flagger and Prometheus'
+# use kustomize to avoid compatibility issues:
+# https://github.com/kubernetes-sigs/kustomize/issues/2390
+# Skipper will throw an Prometheus warning which can be ignored:
+# https://github.com/weaveworks/flagger/issues/664
+kustomize build ${REPO_ROOT}/kustomize/skipper | kubectl apply -f -
+
+kubectl rollout status deployment/skipper-ingress -n kube-system
+kubectl rollout status deployment/flagger-prometheus -n flagger-system
+
+kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n flagger-system rollout status deployment/flagger

test/e2e-workload-ingress.yaml

@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: podinfo-ingress
+  namespace: test
+  labels:
+    app: podinfo
+  annotations:
+    kubernetes.io/ingress.class: skipper
+spec:
+  rules:
+    - host: app.example.com
+      http:
+        paths:
+          - backend:
+              serviceName: podinfo-service
+              servicePort: 80

test/local/skipper.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd $REPO_ROOT
+
+# make test
+make build
+docker tag weaveworks/flagger:latest test/flagger:latest
+make loadtester-build
+(kind get clusters && kubectl delete ns/test --force) || kind create cluster --wait 5m --image kindest/node:v1.16.9
+./test/e2e-skipper.sh
+# port forward prometheus UI to localhost:9090
+kubectl port-forward $(kubectl get pods -l=app=flagger-prometheus -o name -n flagger-system | head -n 1) 9090:9090 -n flagger-system &
+
+./test/e2e-skipper-tests.sh