From 60946e735b16018bcc308b7ae72813a2bb374a8b Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 2 May 2019 14:45:35 +0200 Subject: [PATCH] Provide optional rollback support for HelmReleases --- chart/flux/templates/helm-operator-crd.yaml | 15 ++ cmd/helm-operator/main.go | 10 +- deploy-helm/flux-helm-release-crd.yaml | 15 ++ .../apis/flux.weave.works/v1beta1/types.go | 31 +++- .../v1beta1/zz_generated.deepcopy.go | 23 +++ integrations/helm/chartsync/chartsync.go | 73 +++++---- integrations/helm/operator/operator.go | 24 +++ integrations/helm/release/release.go | 58 ++++++- integrations/helm/status/conditions.go | 77 +++++---- integrations/helm/status/status.go | 152 ++++++++++++------ 10 files changed, 362 insertions(+), 116 deletions(-) diff --git a/chart/flux/templates/helm-operator-crd.yaml b/chart/flux/templates/helm-operator-crd.yaml index 999062650f..c92e17fccc 100644 --- a/chart/flux/templates/helm-operator-crd.yaml +++ b/chart/flux/templates/helm-operator-crd.yaml @@ -44,6 +44,21 @@ spec: type: boolean forceUpgrade: type: boolean + rollback: + type: object + properties: + enable: + type: boolean + force: + type: boolean + recreate: + type: boolean + disableHooks: + type: boolean + timeout: + type: int64 + wait: + type: boolean valueFileSecrets: type: array items: diff --git a/cmd/helm-operator/main.go b/cmd/helm-operator/main.go index f2f80729be..2417207dfa 100644 --- a/cmd/helm-operator/main.go +++ b/cmd/helm-operator/main.go @@ -166,16 +166,16 @@ func main() { TLSHostname: *tillerTLSHostname, }) - // The status updater, to keep track the release status for each - // HelmRelease. It runs as a separate loop for now. - statusUpdater := status.New(ifClient, kubeClient, helmClient, *namespace) - go statusUpdater.Loop(shutdown, log.With(logger, "component", "annotator")) - nsOpt := ifinformers.WithNamespace(*namespace) ifInformerFactory := ifinformers.NewSharedInformerFactoryWithOptions(ifClient, *chartsSyncInterval, nsOpt) fhrInformer := ifInformerFactory.Flux().V1beta1().HelmReleases() go ifInformerFactory.Start(shutdown) + // The status updater, to keep track the release status for each + // HelmRelease. It runs as a separate loop for now. + statusUpdater := status.New(ifClient, fhrInformer.Lister(), helmClient) + go statusUpdater.Loop(shutdown, log.With(logger, "component", "statusupdater")) + queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ChartRelease") // release instance is needed during the sync of git chart changes and during the sync of HelmRelease changes diff --git a/deploy-helm/flux-helm-release-crd.yaml b/deploy-helm/flux-helm-release-crd.yaml index d9bf9a81f5..11dae71ddf 100644 --- a/deploy-helm/flux-helm-release-crd.yaml +++ b/deploy-helm/flux-helm-release-crd.yaml @@ -35,6 +35,21 @@ spec: type: boolean forceUpgrade: type: boolean + rollback: + type: object + properties: + enable: + type: boolean + force: + type: boolean + recreate: + type: boolean + disableHooks: + type: boolean + timeout: + type: int64 + wait: + type: boolean valueFileSecrets: type: array items: diff --git a/integrations/apis/flux.weave.works/v1beta1/types.go b/integrations/apis/flux.weave.works/v1beta1/types.go index d38fbf8994..e744568502 100644 --- a/integrations/apis/flux.weave.works/v1beta1/types.go +++ b/integrations/apis/flux.weave.works/v1beta1/types.go @@ -78,7 +78,8 @@ type GitChartSource struct { SkipDepUpdate bool `json:"skipDepUpdate,omitempty"` } -// DefaultGitRef is the ref assumed if the Ref field is not given in a GitChartSource +// DefaultGitRef is the ref assumed if the Ref field is not given in +// a GitChartSource const DefaultGitRef = "master" func (s GitChartSource) RefOrDefault() string { @@ -103,6 +104,22 @@ func (s RepoChartSource) CleanRepoURL() string { return cleanURL + "/" } +type Rollback struct { + Enable bool `json:"enable,omitempty"` + Force bool `json:"force,omitempty"` + Recreate bool `json:"recreate,omitempty"` + DisableHooks bool `json:"disableHooks,omitempty"` + Timeout *int64 `json:"timeout,omitempty"` + Wait bool `json:"wait,omitempty"` +} + +func (r Rollback) GetTimeout() int64 { + if r.Timeout == nil { + return 300 + } + return *r.Timeout +} + // HelmReleaseSpec is the spec for a HelmRelease resource type HelmReleaseSpec struct { ChartSource `json:"chart"` @@ -119,6 +136,9 @@ type HelmReleaseSpec struct { // Force resource update through delete/recreate, allows recovery from a failed state // +optional ForceUpgrade bool `json:"forceUpgrade,omitempty"` + // Enable rollback and configure options + // +optional + Rollback Rollback `json:"rollback,omitempty"` } // GetTimeout returns the install or upgrade timeout (defaults to 300s) @@ -138,6 +158,10 @@ type HelmReleaseStatus struct { // managed by this resource. ReleaseStatus string `json:"releaseStatus"` + // ObservedGeneration is the most recent generation observed by + // the controller. + ObservedGeneration int64 `json:"observedGeneration"` + // Revision would define what Git hash or Chart version has currently // been deployed. // +optional @@ -155,6 +179,8 @@ type HelmReleaseCondition struct { Type HelmReleaseConditionType `json:"type"` Status v1.ConditionStatus `json:"status"` // +optional + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + // +optional LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` // +optional Reason string `json:"reason,omitempty"` @@ -171,6 +197,9 @@ const ( // Released means the chart release, as specified in this // HelmRelease, has been processed by Helm. HelmReleaseReleased HelmReleaseConditionType = "Released" + // RolledBack means the chart to which the HelmRelease refers + // has been rolled back + HelmReleaseRolledBack HelmReleaseConditionType = "RolledBack" ) // FluxHelmValues embeds chartutil.Values so we can implement deepcopy on map[string]interface{} diff --git a/integrations/apis/flux.weave.works/v1beta1/zz_generated.deepcopy.go b/integrations/apis/flux.weave.works/v1beta1/zz_generated.deepcopy.go index c3cc46ce66..bf121d906c 100644 --- a/integrations/apis/flux.weave.works/v1beta1/zz_generated.deepcopy.go +++ b/integrations/apis/flux.weave.works/v1beta1/zz_generated.deepcopy.go @@ -140,6 +140,7 @@ func (in *HelmRelease) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HelmReleaseCondition) DeepCopyInto(out *HelmReleaseCondition) { *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) return } @@ -209,6 +210,7 @@ func (in *HelmReleaseSpec) DeepCopyInto(out *HelmReleaseSpec) { *out = new(int64) **out = **in } + in.Rollback.DeepCopyInto(&out.Rollback) return } @@ -266,6 +268,27 @@ func (in *RepoChartSource) DeepCopy() *RepoChartSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Rollback) DeepCopyInto(out *Rollback) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(int64) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Rollback. +func (in *Rollback) DeepCopy() *Rollback { + if in == nil { + return nil + } + out := new(Rollback) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ValuesFromSource) DeepCopyInto(out *ValuesFromSource) { *out = *in diff --git a/integrations/helm/chartsync/chartsync.go b/integrations/helm/chartsync/chartsync.go index d474f671a1..2657943537 100644 --- a/integrations/helm/chartsync/chartsync.go +++ b/integrations/helm/chartsync/chartsync.go @@ -42,14 +42,13 @@ import ( "sync" "time" - "k8s.io/apimachinery/pkg/labels" - "github.com/go-kit/kit/log" google_protobuf "github.com/golang/protobuf/ptypes/any" "github.com/google/go-cmp/cmp" "github.com/ncabatoff/go-seq/seq" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -74,6 +73,7 @@ const ( ReasonInstallFailed = "HelmInstallFailed" ReasonDependencyFailed = "UpdateDependencyFailed" ReasonUpgradeFailed = "HelmUgradeFailed" + ReasonRollbackFailed = "HelmRollbackFailed" ReasonCloned = "GitRepoCloned" ReasonSuccess = "HelmSuccess" ) @@ -243,15 +243,15 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn if cloneForChart.export != nil { cloneForChart.export.Clean() } - } - // Enqueue release - cacheKey, err := cache.MetaNamespaceKeyFunc(fhr.GetObjectMeta()) - if err != nil { - continue + // Enqueue release + cacheKey, err := cache.MetaNamespaceKeyFunc(fhr.GetObjectMeta()) + if err != nil { + continue + } + chs.logger.Log("info", "enqueing release upgrade due to change in git chart source", "resource", fhr.ResourceID().String()) + chs.releaseQueue.AddRateLimited(cacheKey) } - chs.logger.Log("info", "enqueing release upgrade due to change in git chart source", "resource", fhr.ResourceID().String()) - chs.releaseQueue.AddRateLimited(cacheKey) } } case <-stopCh: @@ -291,6 +291,8 @@ func (chs *ChartChangeSync) ReconcileReleaseDef(fhr fluxv1beta1.HelmRelease) { // HelmRelease resource, and either installs, upgrades, or does // nothing, depending on the state (or absence) of the release. func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) { + defer chs.updateObservedGeneration(fhr) + releaseName := release.GetReleaseName(fhr) // Attempt to retrieve an upgradable release, in case no release @@ -376,7 +378,7 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) { return } chs.setCondition(fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionTrue, ReasonSuccess, "helm install succeeded") - if err = status.UpdateReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil { + if err = status.SetReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil { chs.logger.Log("warning", "could not update the release revision", "namespace", fhr.Namespace, "resource", fhr.Name, "err", err) } return @@ -404,13 +406,31 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) { return } chs.setCondition(fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionTrue, ReasonSuccess, "helm upgrade succeeded") - if err = status.UpdateReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil { + if err = status.SetReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil { chs.logger.Log("warning", "could not update the release revision", "resource", fhr.ResourceID().String(), "err", err) } return } } +// RollbackRelease rolls back a helm release +func (chs *ChartChangeSync) RollbackRelease(fhr fluxv1beta1.HelmRelease) { + defer chs.updateObservedGeneration(fhr) + + if !fhr.Spec.Rollback.Enable { + return + } + + name := release.GetReleaseName(fhr) + err := chs.release.Rollback(name, fhr.Spec.Rollback.GetTimeout(), fhr.Spec.Rollback.Force, + fhr.Spec.Rollback.Recreate, fhr.Spec.Rollback.DisableHooks, fhr.Spec.Rollback.Wait) + if err != nil { + chs.logger.Log("warning", "unable to rollback chart release", "resource", fhr.ResourceID().String(), "release", name, "err", err) + chs.setCondition(fhr, fluxv1beta1.HelmReleaseRolledBack, v1.ConditionFalse, ReasonRollbackFailed, err.Error()) + } + chs.setCondition(fhr, fluxv1beta1.HelmReleaseRolledBack, v1.ConditionTrue, ReasonSuccess, "helm rollback succeeded") +} + // DeleteRelease deletes the helm release associated with a // HelmRelease. This exists mainly so that the operator code can // call it when it is handling a resource deletion. @@ -464,26 +484,19 @@ func (chs *ChartChangeSync) getCustomResourcesForMirror(mirror string) ([]fluxv1 return fhrs, nil } -// setCondition saves the status of a condition, if it's new -// information. New information is something that adds or changes the -// status, reason or message (i.e., anything but the transition time) -// for one of the types of condition. -func (chs *ChartChangeSync) setCondition(fhr fluxv1beta1.HelmRelease, typ fluxv1beta1.HelmReleaseConditionType, st v1.ConditionStatus, reason, message string) error { - for _, c := range fhr.Status.Conditions { - if c.Type == typ && c.Status == st && c.Message == message && c.Reason == reason { - return nil - } - } +// setCondition saves the status of a condition. +func (chs *ChartChangeSync) setCondition(hr fluxv1beta1.HelmRelease, typ fluxv1beta1.HelmReleaseConditionType, st v1.ConditionStatus, reason, message string) error { + hrClient := chs.ifClient.FluxV1beta1().HelmReleases(hr.Namespace) + condition := status.NewCondition(typ, st, reason, message) + return status.SetCondition(hrClient, hr, condition) +} - fhrClient := chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace) - cond := fluxv1beta1.HelmReleaseCondition{ - Type: typ, - Status: st, - LastTransitionTime: metav1.Now(), - Reason: reason, - Message: message, - } - return status.UpdateConditions(fhrClient, fhr, cond) +// updateObservedGeneration updates the observed generation of the +// given HelmRelease to the generation. +func (chs *ChartChangeSync) updateObservedGeneration(hr fluxv1beta1.HelmRelease) error { + hrClient := chs.ifClient.FluxV1beta1().HelmReleases(hr.Namespace) + + return status.SetObservedGeneration(hrClient, hr, hr.Generation) } func sortStrings(ss []string) []string { diff --git a/integrations/helm/operator/operator.go b/integrations/helm/operator/operator.go index 9c3c300c3d..25b3000f8f 100644 --- a/integrations/helm/operator/operator.go +++ b/integrations/helm/operator/operator.go @@ -24,6 +24,7 @@ import ( fhrv1 "github.com/weaveworks/flux/integrations/client/informers/externalversions/flux.weave.works/v1beta1" iflister "github.com/weaveworks/flux/integrations/client/listers/flux.weave.works/v1beta1" "github.com/weaveworks/flux/integrations/helm/chartsync" + "github.com/weaveworks/flux/integrations/helm/status" ) const ( @@ -233,6 +234,12 @@ func (c *Controller) syncHandler(key string) error { return err } + // (Maybe) attempt a rollback if the release has failed. + if status.ReleaseFailed(*fhr) { + c.sync.RollbackRelease(*fhr) + return nil + } + c.sync.ReconcileReleaseDef(*fhr) c.recorder.Event(fhr, corev1.EventTypeNormal, ChartSynced, MessageChartSynced) return nil @@ -282,6 +289,16 @@ func (c *Controller) enqueueUpdateJob(old, new interface{}) { return } + // Enqueue rollback if the roll-out of the release failed and + // rollbacks are enabled. + if oldFhr.Status.ReleaseStatus != newFhr.Status.ReleaseStatus { + if newFhr.Spec.Rollback.Enable && status.ReleaseFailed(newFhr) { + c.logger.Log("info", "enqueing rollback", "resource", newFhr.ResourceID().String()) + c.enqueueJob(new) + return + } + } + diff := cmp.Diff(oldFhr.Spec, newFhr.Spec) // Filter out any update notifications that are due to status @@ -293,6 +310,13 @@ func (c *Controller) enqueueUpdateJob(old, new interface{}) { return } + // Skip if the current HelmRelease generation has been rolled + // back, as otherwise we will end up in a loop of failure. + if status.HasRolledBack(newFhr) { + c.logger.Log("warning", "release has been rolled back, skipping", "resource", newFhr.ResourceID().String()) + return + } + log := []string{"info", "enqueuing release"} if diff != "" && c.logDiffs { log = append(log, "diff", diff) diff --git a/integrations/helm/release/release.go b/integrations/helm/release/release.go index 2ccb50da31..89bff871c0 100644 --- a/integrations/helm/release/release.go +++ b/integrations/helm/release/release.go @@ -112,11 +112,29 @@ func (r *Release) GetUpgradableRelease(name string) (*hapi_release.Release, erro } } +func (r *Release) mustRollback(name string) (bool, error) { + rls, err := r.HelmClient.ReleaseStatus(name) + if err != nil { + return false, err + } + + status := rls.GetInfo().GetStatus() + switch status.Code { + case hapi_release.Status_FAILED: + r.logger.Log("info", "rolling back release", "release", name) + return true, nil + case hapi_release.Status_PENDING_ROLLBACK: + r.logger.Log("info", "release already has a rollback pending", "release", name) + return false, nil + default: + return false, fmt.Errorf("release with status %s cannot be rolled back", status.Code.String()) + } +} + func (r *Release) canDelete(name string) (bool, error) { rls, err := r.HelmClient.ReleaseStatus(name) if err != nil { - r.logger.Log("error", fmt.Sprintf("Error finding status for release (%s): %#v", name, err)) return false, err } /* @@ -139,7 +157,6 @@ func (r *Release) canDelete(name string) (bool, error) { r.logger.Log("info", fmt.Sprintf("Release %s already deleted", name)) return false, nil default: - r.logger.Log("info", fmt.Sprintf("Release %s with status %s cannot be deleted", name, status.Code.String())) return false, fmt.Errorf("release %s with status %s cannot be deleted", name, status.Code.String()) } } @@ -248,6 +265,43 @@ func (r *Release) Install(chartPath, releaseName string, fhr flux_v1beta1.HelmRe } } +// Rollback rolls back a Chart release if required +func (r *Release) Rollback(name string, timeout int64, force, recreate, disableHooks, wait bool) error { + ok, err := r.mustRollback(name) + if !ok { + if err != nil { + return err + } + return nil + } + + history, err := r.HelmClient.ReleaseHistory(name, k8shelm.WithMaxHistory(5)) + if err != nil { + return err + } + + var version int32 + for _, rls := range history.GetReleases() { + if rls.Info.Status.Code == hapi_release.Status_DEPLOYED { + version = rls.Version + break + } + } + + if version == 0 { + return fmt.Errorf("failed to determine what version to rollback to") + } + + _, err = r.HelmClient.RollbackRelease(name, k8shelm.RollbackVersion(version), k8shelm.RollbackTimeout(timeout), + k8shelm.RollbackForce(force), k8shelm.RollbackRecreate(recreate), k8shelm.RollbackDisableHooks(disableHooks), + k8shelm.RollbackWait(wait), k8shelm.RollbackDescription("Automated rollback by Helm operator")) + if err != nil { + return err + } + r.logger.Log("info", "rolled back release", "release", name, "version", version) + return err +} + // Delete purges a Chart release func (r *Release) Delete(name string) error { ok, err := r.canDelete(name) diff --git a/integrations/helm/status/conditions.go b/integrations/helm/status/conditions.go index 14e892aa55..927e9de43d 100644 --- a/integrations/helm/status/conditions.go +++ b/integrations/helm/status/conditions.go @@ -1,43 +1,66 @@ package status import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/weaveworks/flux/integrations/apis/flux.weave.works/v1beta1" v1beta1client "github.com/weaveworks/flux/integrations/client/clientset/versioned/typed/flux.weave.works/v1beta1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// We can't rely on having UpdateStatus, or strategic merge patching -// for custom resources. So we have to create an object which -// represents the merge path or JSON patch to apply. -func UpdateConditionsPatch(status *v1beta1.HelmReleaseStatus, updates ...v1beta1.HelmReleaseCondition) { - newConditions := make([]v1beta1.HelmReleaseCondition, len(status.Conditions)) - oldConditions := status.Conditions - for i, c := range oldConditions { - newConditions[i] = c - } -updates: - for _, up := range updates { - for i, c := range oldConditions { - if c.Type == up.Type { - newConditions[i] = up - continue updates - } - } - newConditions = append(newConditions, up) + +// NewCondition creates a new HelmReleaseCondition. +func NewCondition(conditionType v1beta1.HelmReleaseConditionType, status v1.ConditionStatus, reason, message string) v1beta1.HelmReleaseCondition { + return v1beta1.HelmReleaseCondition{ + Type: conditionType, + Status: status, + LastUpdateTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, } - status.Conditions = newConditions } -// UpdateConditions retrieves a new copy of the HelmRelease given, -// applies the updates to this copy, and updates the resource in the -// cluster. -func UpdateConditions(client v1beta1client.HelmReleaseInterface, fhr v1beta1.HelmRelease, updates ...v1beta1.HelmReleaseCondition) error { - cFhr, err := client.Get(fhr.Name, v1.GetOptions{}) +// SetCondition updates the HelmRelease to include the given condition. +func SetCondition(client v1beta1client.HelmReleaseInterface, hr v1beta1.HelmRelease, + condition v1beta1.HelmReleaseCondition) error { + + cHr, err := client.Get(hr.Name, metav1.GetOptions{}) if err != nil { return err } - UpdateConditionsPatch(&cFhr.Status, updates...) - _, err = client.UpdateStatus(cFhr) + currCondition := GetCondition(cHr.Status, condition.Type) + if currCondition != nil && currCondition.Status == condition.Status { + condition.LastTransitionTime = currCondition.LastTransitionTime + } + + newConditions := filterOutCondition(cHr.Status.Conditions, condition.Type) + cHr.Status.Conditions = append(newConditions, condition) + + _, err = client.UpdateStatus(cHr) return err } + +// GetCondition returns the condition with the given type. +func GetCondition(status v1beta1.HelmReleaseStatus, conditionType v1beta1.HelmReleaseConditionType) *v1beta1.HelmReleaseCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == conditionType { + return &c + } + } + return nil +} + +// filterOutCondition returns a new slice of conditions without the +// conditions of the given type. +func filterOutCondition(conditions []v1beta1.HelmReleaseCondition, conditionType v1beta1.HelmReleaseConditionType) []v1beta1.HelmReleaseCondition { + var newConditions []v1beta1.HelmReleaseCondition + for _, c := range conditions { + if c.Type == conditionType { + continue + } + newConditions = append(newConditions, c) + } + return newConditions +} diff --git a/integrations/helm/status/status.go b/integrations/helm/status/status.go index 96ca621fb3..7fb9c242cb 100644 --- a/integrations/helm/status/status.go +++ b/integrations/helm/status/status.go @@ -17,12 +17,16 @@ import ( "time" "github.com/go-kit/kit/log" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kube "k8s.io/client-go/kubernetes" "k8s.io/helm/pkg/helm" + helmrelease "k8s.io/helm/pkg/proto/hapi/release" "github.com/weaveworks/flux/integrations/apis/flux.weave.works/v1beta1" - fluxclientset "github.com/weaveworks/flux/integrations/client/clientset/versioned" + ifclientset "github.com/weaveworks/flux/integrations/client/clientset/versioned" + iflister "github.com/weaveworks/flux/integrations/client/listers/flux.weave.works/v1beta1" v1beta1client "github.com/weaveworks/flux/integrations/client/clientset/versioned/typed/flux.weave.works/v1beta1" "github.com/weaveworks/flux/integrations/helm/release" ) @@ -30,22 +34,22 @@ import ( const period = 10 * time.Second type Updater struct { - fluxhelm fluxclientset.Interface + hrClient ifclientset.Interface + hrLister iflister.HelmReleaseLister kube kube.Interface helmClient *helm.Client namespace string } -func New(fhrClient fluxclientset.Interface, kubeClient kube.Interface, helmClient *helm.Client, namespace string) *Updater { +func New(hrClient ifclientset.Interface, hrLister iflister.HelmReleaseLister, helmClient *helm.Client) *Updater { return &Updater{ - fluxhelm: fhrClient, - kube: kubeClient, + hrClient: hrClient, + hrLister: hrLister, helmClient: helmClient, - namespace: namespace, } } -func (a *Updater) Loop(stop <-chan struct{}, logger log.Logger) { +func (u *Updater) Loop(stop <-chan struct{}, logger log.Logger) { ticker := time.NewTicker(period) var logErr error @@ -56,43 +60,23 @@ bail: break bail case <-ticker.C: } - var namespaces []string - if a.namespace != "" { - namespaces = append(namespaces, a.namespace) - } else { - all, err := a.kube.CoreV1().Namespaces().List(metav1.ListOptions{}) - if err != nil { - logErr = err - break bail - } - for _, ns := range all.Items { - namespaces = append(namespaces, ns.Name) - } + list, err := u.hrLister.List(labels.Everything()) + if err != nil { + logErr = err + break bail } - - // Look up HelmReleases - for _, ns := range namespaces { - fhrClient := a.fluxhelm.FluxV1beta1().HelmReleases(ns) - fhrs, err := fhrClient.List(metav1.ListOptions{}) - if err != nil { - logErr = err - break bail + for _, hr := range list { + nsHrClient := u.hrClient.FluxV1beta1().HelmReleases(hr.Namespace) + releaseName := release.GetReleaseName(*hr) + releaseStatus, _ := u.helmClient.ReleaseStatus(releaseName) + // If we are unable to get the status, we do not care why + if releaseStatus == nil { + continue } - for _, fhr := range fhrs.Items { - releaseName := release.GetReleaseName(fhr) - // If we don't get the content, we don't care why - content, _ := a.helmClient.ReleaseContent(releaseName) - if content == nil { - continue - } - status := content.GetRelease().GetInfo().GetStatus() - if status.GetCode().String() != fhr.Status.ReleaseStatus { - err := UpdateReleaseStatus(fhrClient, fhr, releaseName, status.GetCode().String()) - if err != nil { - logger.Log("namespace", ns, "resource", fhr.Name, "err", err) - continue - } - } + statusStr := releaseStatus.Info.Status.Code.String() + if err := SetReleaseStatus(nsHrClient, *hr, releaseName, statusStr); err != nil { + logger.Log("namespace", hr.Namespace, "resource", hr.Name, "err", err) + continue } } } @@ -101,25 +85,91 @@ bail: logger.Log("loop", "stopping", "err", logErr) } -func UpdateReleaseStatus(client v1beta1client.HelmReleaseInterface, fhr v1beta1.HelmRelease, releaseName, releaseStatus string) error { - cFhr, err := client.Get(fhr.Name, metav1.GetOptions{}) + +// SetReleaseStatus updates the status of the HelmRelease to the given +// release name and/or release status. +func SetReleaseStatus(client v1beta1client.HelmReleaseInterface, hr v1beta1.HelmRelease, releaseName, releaseStatus string) error { + cHr, err := client.Get(hr.Name, metav1.GetOptions{}) if err != nil { return err } - cFhr.Status.ReleaseName = releaseName - cFhr.Status.ReleaseStatus = releaseStatus - _, err = client.UpdateStatus(cFhr) + if cHr.Status.ReleaseName == releaseName && cHr.Status.ReleaseStatus == releaseStatus { + return nil + } + + cHr.Status.ReleaseName = releaseName + cHr.Status.ReleaseStatus = releaseStatus + + _, err = client.UpdateStatus(cHr) return err } -func UpdateReleaseRevision(client v1beta1client.HelmReleaseInterface, fhr v1beta1.HelmRelease, revision string) error { - cFhr, err := client.Get(fhr.Name, metav1.GetOptions{}) + +// SetReleaseRevision updates the status of the HelmRelease to the +// given revision. +func SetReleaseRevision(client v1beta1client.HelmReleaseInterface, hr v1beta1.HelmRelease, revision string) error { + cHr, err := client.Get(hr.Name, metav1.GetOptions{}) if err != nil { return err } - cFhr.Status.Revision = revision - _, err = client.UpdateStatus(cFhr) + if cHr.Status.Revision == revision { + return nil + } + + cHr.Status.Revision = revision + + _, err = client.UpdateStatus(cHr) return err } + +// SetObservedGeneration updates the observed generation status of the +// HelmRelease to the given generation. +func SetObservedGeneration(client v1beta1client.HelmReleaseInterface, hr v1beta1.HelmRelease, generation int64) error { + cHr, err := client.Get(hr.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + if cHr.Status.ObservedGeneration >= generation { + return nil + } + + cHr.Status.ObservedGeneration = generation + + _, err = client.UpdateStatus(cHr) + return err +} + +// ReleaseFailed returns if the roll-out of the HelmRelease failed. +func ReleaseFailed(hr v1beta1.HelmRelease) bool { + return hr.Status.ReleaseStatus == helmrelease.Status_FAILED.String() +} + + +// HasSynced returns if the HelmRelease has been processed by the +// controller. +func HasSynced(hr v1beta1.HelmRelease) bool { + return hr.Status.ObservedGeneration >= hr.Generation +} + +// HasRolledBack returns if the current generation of the HelmRelease +// has been rolled back. +func HasRolledBack(hr v1beta1.HelmRelease) bool { + if !HasSynced(hr) { + return false + } + + rolledBack := GetCondition(hr.Status, v1beta1.HelmReleaseRolledBack) + if rolledBack == nil { + return false + } + + chartFetched := GetCondition(hr.Status, v1beta1.HelmReleaseChartFetched) + if chartFetched != nil { + return !(chartFetched.Status == v1.ConditionTrue && rolledBack.LastUpdateTime.Before(&chartFetched.LastUpdateTime)) + } + + return rolledBack.Status == v1.ConditionTrue +}