Provide optional rollback support for HelmReleases

chart/flux/templates/helm-operator-crd.yaml

@@ -44,6 +44,21 @@ spec:
               type: boolean
             forceUpgrade:
               type: boolean
+            rollback:
+              type: object
+              properties:
+                enable:
+                  type: boolean
+                force:
+                  type: boolean
+                recreate:
+                  type: boolean
+                disableHooks:
+                  type: boolean
+                timeout:
+                  type: int64
+                wait:
+                  type: boolean
             valueFileSecrets:
               type: array
               items:

cmd/helm-operator/main.go

@@ -166,16 +166,16 @@ func main() {
 		TLSHostname: *tillerTLSHostname,
 	})
 
-	// The status updater, to keep track the release status for each
-	// HelmRelease. It runs as a separate loop for now.
-	statusUpdater := status.New(ifClient, kubeClient, helmClient, *namespace)
-	go statusUpdater.Loop(shutdown, log.With(logger, "component", "annotator"))
-
 	nsOpt := ifinformers.WithNamespace(*namespace)
 	ifInformerFactory := ifinformers.NewSharedInformerFactoryWithOptions(ifClient, *chartsSyncInterval, nsOpt)
 	fhrInformer := ifInformerFactory.Flux().V1beta1().HelmReleases()
 	go ifInformerFactory.Start(shutdown)
 
+	// The status updater, to keep track the release status for each
+	// HelmRelease. It runs as a separate loop for now.
+	statusUpdater := status.New(ifClient, fhrInformer.Lister(), helmClient)
+	go statusUpdater.Loop(shutdown, log.With(logger, "component", "statusupdater"))
+
 	queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ChartRelease")
 
 	// release instance is needed during the sync of git chart changes and during the sync of HelmRelease changes

deploy-helm/flux-helm-release-crd.yaml

@@ -35,6 +35,21 @@ spec:
               type: boolean
             forceUpgrade:
               type: boolean
+            rollback:
+              type: object
+              properties:
+                enable:
+                  type: boolean
+                force:
+                  type: boolean
+                recreate:
+                  type: boolean
+                disableHooks:
+                  type: boolean
+                timeout:
+                  type: int64
+                wait:
+                  type: boolean
             valueFileSecrets:
               type: array
               items:

integrations/apis/flux.weave.works/v1beta1/types.go

@@ -78,7 +78,8 @@ type GitChartSource struct {
 	SkipDepUpdate bool `json:"skipDepUpdate,omitempty"`
 }
 
-// DefaultGitRef is the ref assumed if the Ref field is not given in a GitChartSource
+// DefaultGitRef is the ref assumed if the Ref field is not given in
+// a GitChartSource
 const DefaultGitRef = "master"
 
 func (s GitChartSource) RefOrDefault() string {
@@ -103,6 +104,22 @@ func (s RepoChartSource) CleanRepoURL() string {
 	return cleanURL + "/"
 }
 
+type Rollback struct {
+	Enable 		 bool 	`json:"enable,omitempty"`
+	Force  		 bool 	`json:"force,omitempty"`
+	Recreate 	 bool 	`json:"recreate,omitempty"`
+	DisableHooks bool 	`json:"disableHooks,omitempty"`
+	Timeout 	 *int64 `json:"timeout,omitempty"`
+	Wait 		 bool   `json:"wait,omitempty"`
+}
+
+func (r Rollback) GetTimeout() int64 {
+	if r.Timeout == nil {
+		return 300
+	}
+	return *r.Timeout
+}
+
 // HelmReleaseSpec is the spec for a HelmRelease resource
 type HelmReleaseSpec struct {
 	ChartSource      `json:"chart"`
@@ -119,6 +136,9 @@ type HelmReleaseSpec struct {
 	// Force resource update through delete/recreate, allows recovery from a failed state
 	// +optional
 	ForceUpgrade bool `json:"forceUpgrade,omitempty"`
+	// Enable rollback and configure options
+	// +optional
+	Rollback Rollback `json:"rollback,omitempty"`
 }
 
 // GetTimeout returns the install or upgrade timeout (defaults to 300s)
@@ -138,6 +158,10 @@ type HelmReleaseStatus struct {
 	// managed by this resource.
 	ReleaseStatus string `json:"releaseStatus"`
 
+	// ObservedGeneration is the most recent generation observed by
+	// the controller.
+	ObservedGeneration int64 `json:"observedGeneration"`
+
 	// Revision would define what Git hash or Chart version has currently
 	// been deployed.
 	// +optional
@@ -155,6 +179,8 @@ type HelmReleaseCondition struct {
 	Type   HelmReleaseConditionType `json:"type"`
 	Status v1.ConditionStatus       `json:"status"`
 	// +optional
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+	// +optional
 	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
 	// +optional
 	Reason string `json:"reason,omitempty"`
@@ -171,6 +197,9 @@ const (
 	// Released means the chart release, as specified in this
 	// HelmRelease, has been processed by Helm.
 	HelmReleaseReleased HelmReleaseConditionType = "Released"
+	// RolledBack means the chart to which the HelmRelease refers
+	// has been rolled back
+	HelmReleaseRolledBack HelmReleaseConditionType = "RolledBack"
 )
 
 // FluxHelmValues embeds chartutil.Values so we can implement deepcopy on map[string]interface{}

integrations/helm/chartsync/chartsync.go

@@ -42,14 +42,13 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/apimachinery/pkg/labels"
-
 	"github.com/go-kit/kit/log"
 	google_protobuf "github.com/golang/protobuf/ptypes/any"
 	"github.com/google/go-cmp/cmp"
 	"github.com/ncabatoff/go-seq/seq"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
@@ -74,6 +73,7 @@ const (
 	ReasonInstallFailed    = "HelmInstallFailed"
 	ReasonDependencyFailed = "UpdateDependencyFailed"
 	ReasonUpgradeFailed    = "HelmUgradeFailed"
+	ReasonRollbackFailed   = "HelmRollbackFailed"
 	ReasonCloned           = "GitRepoCloned"
 	ReasonSuccess          = "HelmSuccess"
 )
@@ -243,15 +243,15 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn
 							if cloneForChart.export != nil {
 								cloneForChart.export.Clean()
 							}
-						}
 
-						// Enqueue release
-						cacheKey, err := cache.MetaNamespaceKeyFunc(fhr.GetObjectMeta())
-						if err != nil {
-							continue
+							// Enqueue release
+							cacheKey, err := cache.MetaNamespaceKeyFunc(fhr.GetObjectMeta())
+							if err != nil {
+								continue
+							}
+							chs.logger.Log("info", "enqueing release upgrade due to change in git chart source", "resource", fhr.ResourceID().String())
+							chs.releaseQueue.AddRateLimited(cacheKey)
 						}
-						chs.logger.Log("info", "enqueing release upgrade due to change in git chart source", "resource", fhr.ResourceID().String())
-						chs.releaseQueue.AddRateLimited(cacheKey)
 					}
 				}
 			case <-stopCh:
@@ -291,6 +291,8 @@ func (chs *ChartChangeSync) ReconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
 // HelmRelease resource, and either installs, upgrades, or does
 // nothing, depending on the state (or absence) of the release.
 func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
+	defer chs.updateObservedGeneration(fhr)
+
 	releaseName := release.GetReleaseName(fhr)
 
 	// Attempt to retrieve an upgradable release, in case no release
@@ -376,7 +378,7 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
 			return
 		}
 		chs.setCondition(fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionTrue, ReasonSuccess, "helm install succeeded")
-		if err = status.UpdateReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil {
+		if err = status.SetReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil {
 			chs.logger.Log("warning", "could not update the release revision", "namespace", fhr.Namespace, "resource", fhr.Name, "err", err)
 		}
 		return
@@ -404,13 +406,31 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
 			return
 		}
 		chs.setCondition(fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionTrue, ReasonSuccess, "helm upgrade succeeded")
-		if err = status.UpdateReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil {
+		if err = status.SetReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil {
 			chs.logger.Log("warning", "could not update the release revision", "resource", fhr.ResourceID().String(), "err", err)
 		}
 		return
 	}
 }
 
+// RollbackRelease rolls back a helm release
+func (chs *ChartChangeSync) RollbackRelease(fhr fluxv1beta1.HelmRelease) {
+	defer chs.updateObservedGeneration(fhr)
+
+	if !fhr.Spec.Rollback.Enable {
+		return
+	}
+
+	name := release.GetReleaseName(fhr)
+	err := chs.release.Rollback(name, fhr.Spec.Rollback.GetTimeout(), fhr.Spec.Rollback.Force,
+		fhr.Spec.Rollback.Recreate, fhr.Spec.Rollback.DisableHooks, fhr.Spec.Rollback.Wait)
+	if err != nil {
+		chs.logger.Log("warning", "unable to rollback chart release", "resource", fhr.ResourceID().String(), "release", name, "err", err)
+		chs.setCondition(fhr, fluxv1beta1.HelmReleaseRolledBack, v1.ConditionFalse, ReasonRollbackFailed, err.Error())
+	}
+	chs.setCondition(fhr, fluxv1beta1.HelmReleaseRolledBack, v1.ConditionTrue, ReasonSuccess, "helm rollback succeeded")
+}
+
 // DeleteRelease deletes the helm release associated with a
 // HelmRelease. This exists mainly so that the operator code can
 // call it when it is handling a resource deletion.
@@ -464,26 +484,19 @@ func (chs *ChartChangeSync) getCustomResourcesForMirror(mirror string) ([]fluxv1
 	return fhrs, nil
 }
 
-// setCondition saves the status of a condition, if it's new
-// information. New information is something that adds or changes the
-// status, reason or message (i.e., anything but the transition time)
-// for one of the types of condition.
-func (chs *ChartChangeSync) setCondition(fhr fluxv1beta1.HelmRelease, typ fluxv1beta1.HelmReleaseConditionType, st v1.ConditionStatus, reason, message string) error {
-	for _, c := range fhr.Status.Conditions {
-		if c.Type == typ && c.Status == st && c.Message == message && c.Reason == reason {
-			return nil
-		}
-	}
+// setCondition saves the status of a condition.
+func (chs *ChartChangeSync) setCondition(hr fluxv1beta1.HelmRelease, typ fluxv1beta1.HelmReleaseConditionType, st v1.ConditionStatus, reason, message string) error {
+	hrClient := chs.ifClient.FluxV1beta1().HelmReleases(hr.Namespace)
+	condition := status.NewCondition(typ, st, reason, message)
+	return status.SetCondition(hrClient, hr, condition)
+}
 
-	fhrClient := chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace)
-	cond := fluxv1beta1.HelmReleaseCondition{
-		Type:               typ,
-		Status:             st,
-		LastTransitionTime: metav1.Now(),
-		Reason:             reason,
-		Message:            message,
-	}
-	return status.UpdateConditions(fhrClient, fhr, cond)
+// updateObservedGeneration updates the observed generation of the
+// given HelmRelease to the generation.
+func (chs *ChartChangeSync) updateObservedGeneration(hr fluxv1beta1.HelmRelease) error {
+	hrClient := chs.ifClient.FluxV1beta1().HelmReleases(hr.Namespace)
+
+	return status.SetObservedGeneration(hrClient, hr, hr.Generation)
 }
 
 func sortStrings(ss []string) []string {

integrations/helm/operator/operator.go

@@ -24,6 +24,7 @@ import (
 	fhrv1 "github.com/weaveworks/flux/integrations/client/informers/externalversions/flux.weave.works/v1beta1"
 	iflister "github.com/weaveworks/flux/integrations/client/listers/flux.weave.works/v1beta1"
 	"github.com/weaveworks/flux/integrations/helm/chartsync"
+	"github.com/weaveworks/flux/integrations/helm/status"
 )
 
 const (
@@ -233,6 +234,12 @@ func (c *Controller) syncHandler(key string) error {
 		return err
 	}
 
+	// (Maybe) attempt a rollback if the release has failed.
+	if status.ReleaseFailed(*fhr) {
+		c.sync.RollbackRelease(*fhr)
+		return nil
+	}
+
 	c.sync.ReconcileReleaseDef(*fhr)
 	c.recorder.Event(fhr, corev1.EventTypeNormal, ChartSynced, MessageChartSynced)
 	return nil
@@ -282,6 +289,16 @@ func (c *Controller) enqueueUpdateJob(old, new interface{}) {
 		return
 	}
 
+	// Enqueue rollback if the roll-out of the release failed and
+	// rollbacks are enabled.
+	if oldFhr.Status.ReleaseStatus != newFhr.Status.ReleaseStatus {
+		if newFhr.Spec.Rollback.Enable && status.ReleaseFailed(newFhr) {
+			c.logger.Log("info", "enqueing rollback", "resource", newFhr.ResourceID().String())
+			c.enqueueJob(new)
+			return
+		}
+	}
+
 	diff := cmp.Diff(oldFhr.Spec, newFhr.Spec)
 
 	// Filter out any update notifications that are due to status
@@ -293,6 +310,13 @@ func (c *Controller) enqueueUpdateJob(old, new interface{}) {
 		return
 	}
 
+	// Skip if the current HelmRelease generation has been rolled
+	// back, as otherwise we will end up in a loop of failure.
+	if status.HasRolledBack(newFhr) {
+		c.logger.Log("warning", "release has been rolled back, skipping", "resource", newFhr.ResourceID().String())
+		return
+	}
+
 	log := []string{"info", "enqueuing release"}
 	if diff != "" && c.logDiffs {
 		log = append(log, "diff", diff)