From 49ad41c83702e99dd01b5b7e8260d81cb4ff1406 Mon Sep 17 00:00:00 2001 From: Shubham Bajpai <shubham.bajpai@mayadata.io> Date: Wed, 11 Mar 2020 20:16:45 +0530 Subject: [PATCH] refact(upgrade): scale down jiva target deploy before replica patch (#1626) Avoid the replica's from getting connected to the older version of the controller and triggering a snapshot delete and so forth code. Signed-off-by: shubham <shubham.bajpai@mayadata.io> --- .../deployment/appsv1/v1alpha1/kubernetes.go | 50 ++++++++++++++++- pkg/upgrade/templates/v1/jiva_target.go | 55 ++++++++++--------- pkg/upgrade/upgrader/jiva_upgrade.go | 52 ++++++++++++++++++ 3 files changed, 129 insertions(+), 28 deletions(-) diff --git a/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go b/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go index bcc0d176ec..6427224cca 100644 --- a/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go +++ b/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go @@ -17,9 +17,10 @@ package v1alpha1 import ( "encoding/json" - "github.com/openebs/maya/pkg/debug" "strings" + "github.com/openebs/maya/pkg/debug" + client "github.com/openebs/maya/pkg/kubernetes/client/v1alpha1" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -61,6 +62,14 @@ type createFn func( deploy *appsv1.Deployment, ) (*appsv1.Deployment, error) +// updateFn is a typed function that abstracts +// updating a deployment instance in kubernetes cluster +type updateFn func( + cli *kubernetes.Clientset, + namespace string, + deploy *appsv1.Deployment, +) (*appsv1.Deployment, error) + // deleteFn is a typed function that abstracts // deleting a deployment from kubernetes cluster type deleteFn func( @@ -137,6 +146,17 @@ func defaultCreate( return cli.AppsV1().Deployments(namespace).Create(deploy) } +// defaultUpdate is the default implementation to update +// a deployment instance in kubernetes cluster +func defaultUpdate( + cli *kubernetes.Clientset, + namespace string, + deploy *appsv1.Deployment, +) (*appsv1.Deployment, error) { + + return cli.AppsV1().Deployments(namespace).Update(deploy) +} + // defaultDel is the default implementation to delete a // deployment instance in kubernetes cluster func defaultDel( @@ -193,6 +213,7 @@ type Kubeclient struct { get getFn list listFn create createFn + update updateFn del deleteFn patch patchFn rolloutStatus rolloutStatusFn @@ -222,6 +243,9 @@ func (k *Kubeclient) withDefaults() { if k.create == nil { k.create = defaultCreate } + if k.update == nil { + k.update = defaultUpdate + } if k.del == nil { k.del = defaultDel } @@ -394,6 +418,30 @@ func (k *Kubeclient) Create(deployment *appsv1.Deployment) (*appsv1.Deployment, return k.create(cli, k.namespace, deployment) } +// Update updates a deployment in specified namespace in kubernetes cluster +func (k *Kubeclient) Update(deployment *appsv1.Deployment) (*appsv1.Deployment, error) { + + if debug.EI.IsDeploymentUpdateErrorInjected() { + return nil, errors.New("Deployment update error via injection") + } + + if deployment == nil { + return nil, errors.New("failed to update deployment: nil deployment object") + } + + cli, err := k.getClientOrCached() + if err != nil { + return nil, errors.Wrapf( + err, + "failed to update deployment {%s} in namespace {%s}", + deployment.Name, + deployment.Namespace, + ) + } + + return k.update(cli, k.namespace, deployment) +} + // RolloutStatusf returns deployment's rollout status for given name // in raw bytes func (k *Kubeclient) RolloutStatusf(name string) (op []byte, err error) { diff --git a/pkg/upgrade/templates/v1/jiva_target.go b/pkg/upgrade/templates/v1/jiva_target.go index a45fdd9088..ce7a4775d8 100644 --- a/pkg/upgrade/templates/v1/jiva_target.go +++ b/pkg/upgrade/templates/v1/jiva_target.go @@ -19,31 +19,32 @@ package templates var ( // JivaTargetPatch is generic template for target patch JivaTargetPatch = `{ - "metadata": { - "labels": { - "openebs.io/version": "{{.UpgradeVersion}}" - } - }, - "spec": { - "template": { - "metadata": { - "labels":{ - "openebs.io/version": "{{.UpgradeVersion}}" - } - }, - "spec": { - "containers": [ - { - "name": "{{.ControllerContainerName}}", - "image": "{{.ControllerImage}}:{{.ImageTag}}" - }, - { - "name": "maya-volume-exporter", - "image": "{{.MExporterImage}}:{{.ImageTag}}" - } - ] - } - } - } - }` + "metadata": { + "labels": { + "openebs.io/version": "{{.UpgradeVersion}}" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "labels": { + "openebs.io/version": "{{.UpgradeVersion}}" + } + }, + "spec": { + "containers": [ + { + "name": "{{.ControllerContainerName}}", + "image": "{{.ControllerImage}}:{{.ImageTag}}" + }, + { + "name": "maya-volume-exporter", + "image": "{{.MExporterImage}}:{{.ImageTag}}" + } + ] + } + } + } +}` ) diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index e70c2b5c1a..d85a3f9c05 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -29,6 +29,7 @@ import ( templates "github.com/openebs/maya/pkg/upgrade/templates/v1" errors "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" k8serror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -419,6 +420,45 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } +func scaleDownTargetDeploy(name, namespace string) error { + klog.Infof("Scaling down target deploy %s in %s namespace", name, namespace) + deployObj, err := deployClient.WithNamespace(namespace).Get(name) + if err != nil { + return err + } + pvLabelKey := "openebs.io/persistent-volume" + pvName := deployObj.Labels[pvLabelKey] + controllerLabel := "openebs.io/controller=jiva-controller," + + pvLabelKey + "=" + pvName + var zero int32 + deployObj.Spec.Replicas = &zero + _, err = deployClient.WithNamespace(namespace).Update(deployObj) + if err != nil { + return err + } + podList := &corev1.PodList{} + // Wait for up to 5 minutes for target pod to go away. + for i := 0; i < 60; i++ { + podList, err = podClient.WithNamespace(namespace).List( + metav1.ListOptions{ + LabelSelector: controllerLabel, + }) + if err != nil { + return err + } + if len(podList.Items) > 0 { + time.Sleep(time.Second * 5) + } else { + break + } + } + // If pod is not deleted within 5 minutes return error. + if len(podList.Items) > 0 { + return errors.Errorf("target pod still present") + } + return nil +} + func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { var err, uerr error statusObj := utask.UpgradeDetailedStatuses{Step: utask.ReplicaUpgrade} @@ -429,6 +469,18 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { } statusObj.Phase = utask.StepErrored + + err = scaleDownTargetDeploy(j.controllerObj.name, j.ns) + if err != nil { + statusObj.Message = "failed to scale down target depoyment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return errors.Wrap(err, "failed to scale down target depoyment") + } + // replica patch err = patchReplica(j.replicaObj, j.ns) if err != nil {