Implement confirm rollout gate, hook and API

artifacts/ab-testing/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/appmesh/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/canaries/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/flagger/crd.yaml

@@ -174,6 +174,7 @@ spec:
                 - ""
                 - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
                 - Succeeded
                 - Failed

artifacts/gloo/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/loadtester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.5.0
+          image: weaveworks/flagger-loadtester:0.6.1
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

artifacts/nginx/deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/workloads/canary-deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/workloads/primary-deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.1
+        image: quay.io/stefanprodan/podinfo:1.7.1
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

charts/flagger/templates/crd.yaml

@@ -175,6 +175,7 @@ spec:
                 - ""
                 - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
                 - Succeeded
                 - Failed

charts/loadtester/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: loadtester
-version: 0.5.0
-appVersion: 0.5.0
+version: 0.6.0
+appVersion: 0.6.1
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook.

charts/loadtester/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 
 image:
   repository: weaveworks/flagger-loadtester
-  tag: 0.4.0
+  tag: 0.6.1
   pullPolicy: IfNotPresent
 
 logLevel: info

charts/podinfo/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-version: 2.2.0
-appVersion: 1.4.0
+version: 2.3.0
+appVersion: 1.7.0
 name: podinfo
 engine: gotpl
 description: Flagger canary deployment demo chart

charts/podinfo/values.yaml

@@ -1,7 +1,7 @@
 # Default values for podinfo.
 image:
   repository: quay.io/stefanprodan/podinfo
-  tag: 1.4.0
+  tag: 1.7.0
   pullPolicy: IfNotPresent
 
 service:

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var VERSION = "0.5.0"
+var VERSION = "0.6.1"
 var (
 	logLevel          string
 	port              string
@@ -47,5 +47,7 @@ func main() {
 	go taskRunner.Start(100*time.Millisecond, stopCh)
 
 	logger.Infof("Starting load tester v%s API on port %s", VERSION, port)
-	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, stopCh)
+
+	gateStorage := loadtester.NewGateStorage("in-memory")
+	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, gateStorage, stopCh)
 }

docs/gitbook/how-it-works.md

@@ -105,6 +105,53 @@ convention you can specify your label with the `-selector-labels` flag.
 The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and 
 the Istio Virtual Service. The container port from the target deployment should match the `service.port` value.
 
+### Canary status
+
+Get the current status of canary deployments cluster wide: 
+
+```bash
+kubectl get canaries --all-namespaces
+
+NAMESPACE   NAME      STATUS        WEIGHT   LASTTRANSITIONTIME
+test        podinfo   Progressing   15       2019-06-30T14:05:07Z
+prod        frontend  Succeeded     0        2019-06-30T16:15:07Z
+prod        backend   Failed        0        2019-06-30T17:05:07Z
+```
+
+The status condition reflects the last know state of the canary analysis:
+
+```bash
+kubectl -n test get canary/podinfo -oyaml | awk '/status/,0'
+```
+
+A successful rollout status:
+
+```yaml
+status:
+  canaryWeight: 0
+  failedChecks: 0
+  iterations: 0
+  lastAppliedSpec: "14788816656920327485"
+  lastPromotedSpec: "14788816656920327485"
+  conditions:
+  - lastTransitionTime: "2019-07-10T08:23:18Z"
+    lastUpdateTime: "2019-07-10T08:23:18Z"
+    message: Canary analysis completed successfully, promotion finished.
+    reason: Succeeded
+    status: "True"
+    type: Promoted
+```
+
+The `Promoted` status condition can have one of the following reasons: Initialized, Waiting, Progressing, Succeeded or Failed.
+A failed canary will have the promoted status set to `false`,
+the reason to `failed` and the last applied spec will be different to the last promoted one.
+
+Wait for a successful rollout:
+
+```bash
+kubectl wait canary/podinfo --for=condition=promoted
+```
+
 ### Istio routing
 
 Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. 
@@ -608,6 +655,8 @@ The canary analysis can be extended with webhooks. Flagger will call each webhoo
 determine from the response status code (HTTP 2xx) if the canary is failing or not.
 
 There are three types of hooks:
+* Confirm-rollout hooks are executed before scaling up the canary deployment and ca be used for manual approval.
+The rollout is paused until the hook returns a successful HTTP status code.
 * Pre-rollout hooks are executed before routing traffic to canary. 
 The canary advancement is paused if a pre-rollout hook fails and if the number of failures reach the 
 threshold the canary will be rollback.
@@ -621,6 +670,9 @@ Spec:
 ```yaml
   canaryAnalysis:
     webhooks:
+      - name: "start gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
       - name: "smoke test"
         type: pre-rollout
         url: http://flagger-helmtester.kube-system/
@@ -832,3 +884,66 @@ As an alternative to Helm you can use the [Bash Automated Testing System](https:
 ```
 
 Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container.
+
+### Manual Gating
+
+For manual approval of a canary deployment you can use the `confirm-rollout` webhook. 
+The confirmation hooks are executed before the pre-rollout hooks. 
+Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
+
+Manual gating with Flagger's tester:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/halt
+```
+
+The `/gate/halt` returns HTTP 403 thus blocking the rollout. 
+
+If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval.
+
+Change the URL to `/gate/approve` to start the canary analysis:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "start gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
+```
+
+Manual gating can be driven with Flagger's tester API. Set the confirmation URL to `/gate/check`:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "ask confirmation"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/check
+```
+
+By default the gate is closed, you can start or resume the canary rollout with:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xxxx-xxxx sh
+
+curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/open 
+```
+
+You can pause the rollout at any time with:
+
+```bash
+curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close 
+```
+
+If a canary analysis is paused the status will change to waiting:
+
+```bash
+kubectl get canary/podinfo
+
+NAME      STATUS        WEIGHT
+podinfo   Waiting       0
+```

docs/gitbook/tutorials/canary-helm-gitops.md

@@ -344,7 +344,7 @@ launch the `frontend` and `backend` apps.
 A CI/CD pipeline for the `frontend` release could look like this:
 
 * cut a release from the master branch of the podinfo code repo with the git tag `1.4.1`
-* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry
+* CI builds the image and pushes the `podinfo:1.7.1` image to the container registry
 * Flux scans the registry and updates the Helm release `image.tag` to `1.4.1`
 * Flux commits and push the change to the cluster repo
 * Flux applies the updated Helm release on the cluster

docs/gitbook/tutorials/flagger-smi-istio.md

@@ -236,7 +236,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/ab-testing.md

@@ -131,7 +131,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/abtest \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/appmesh-progressive-delivery.md

@@ -176,7 +176,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/blue-green.md

@@ -59,8 +59,8 @@ Create a deployment and a horizontal pod autoscaler:
 ```bash
 export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master
 
-kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml
-kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml
+kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml
+kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml
 ```
 
 Deploy the load testing service to generate traffic during the analysis:
@@ -172,7 +172,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -297,7 +297,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=quay.io/stefanprodan/podinfo:1.7.3
 ```
 
 Generate 404s:

docs/gitbook/usage/gloo-progressive-delivery.md

@@ -197,7 +197,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/linkerd-progressive-delivery.md

@@ -165,7 +165,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/nginx-progressive-delivery.md

@@ -189,7 +189,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/progressive-delivery.md

@@ -119,7 +119,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

kustomize/base/flagger/crd.yaml

@@ -174,6 +174,7 @@ spec:
                 - ""
                 - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
                 - Succeeded
                 - Failed