From cce307ba5427500a062b6b7925134b20b25ee243 Mon Sep 17 00:00:00 2001 From: shubham Date: Tue, 10 Mar 2020 22:14:02 +0530 Subject: [PATCH 1/4] refact(upgrade): scale down jiva target deploy before replica patch Signed-off-by: shubham --- .../deployment/appsv1/v1alpha1/kubernetes.go | 50 ++++++++++++++++- pkg/upgrade/templates/v1/jiva_target.go | 55 ++++++++++--------- pkg/upgrade/upgrader/jiva_upgrade.go | 50 +++++++++++++++++ 3 files changed, 127 insertions(+), 28 deletions(-) diff --git a/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go b/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go index bcc0d176ec..a48e6b0fc3 100644 --- a/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go +++ b/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go @@ -17,9 +17,10 @@ package v1alpha1 import ( "encoding/json" - "github.com/openebs/maya/pkg/debug" "strings" + "github.com/openebs/maya/pkg/debug" + client "github.com/openebs/maya/pkg/kubernetes/client/v1alpha1" "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -61,6 +62,14 @@ type createFn func( deploy *appsv1.Deployment, ) (*appsv1.Deployment, error) +// updateFn is a typed function that abstracts +// updating a deployment instance in kubernetes cluster +type updateFn func( + cli *kubernetes.Clientset, + namespace string, + deploy *appsv1.Deployment, +) (*appsv1.Deployment, error) + // deleteFn is a typed function that abstracts // deleting a deployment from kubernetes cluster type deleteFn func( @@ -137,6 +146,17 @@ func defaultCreate( return cli.AppsV1().Deployments(namespace).Create(deploy) } +// defaultUpdate is the default implementation to update +// a deployment instance in kubernetes cluster +func defaultUpdate( + cli *kubernetes.Clientset, + namespace string, + deploy *appsv1.Deployment, +) (*appsv1.Deployment, error) { + + return cli.AppsV1().Deployments(namespace).Update(deploy) +} + // defaultDel is the default implementation to delete a // deployment instance in kubernetes cluster func defaultDel( @@ -193,6 +213,7 @@ type Kubeclient struct { get getFn list listFn create createFn + update updateFn del deleteFn patch patchFn rolloutStatus rolloutStatusFn @@ -222,6 +243,9 @@ func (k *Kubeclient) withDefaults() { if k.create == nil { k.create = defaultCreate } + if k.update == nil { + k.update = defaultUpdate + } if k.del == nil { k.del = defaultDel } @@ -394,6 +418,30 @@ func (k *Kubeclient) Create(deployment *appsv1.Deployment) (*appsv1.Deployment, return k.create(cli, k.namespace, deployment) } +// Update updates a deployment in specified namespace in kubernetes cluster +func (k *Kubeclient) Update(deployment *appsv1.Deployment) (*appsv1.Deployment, error) { + + if debug.EI.IsDeploymentCreateErrorInjected() { + return nil, errors.New("Deployment update error via injection") + } + + if deployment == nil { + return nil, errors.New("failed to update deployment: nil deployment object") + } + + cli, err := k.getClientOrCached() + if err != nil { + return nil, errors.Wrapf( + err, + "failed to update deployment {%s} in namespace {%s}", + deployment.Name, + deployment.Namespace, + ) + } + + return k.update(cli, k.namespace, deployment) +} + // RolloutStatusf returns deployment's rollout status for given name // in raw bytes func (k *Kubeclient) RolloutStatusf(name string) (op []byte, err error) { diff --git a/pkg/upgrade/templates/v1/jiva_target.go b/pkg/upgrade/templates/v1/jiva_target.go index a45fdd9088..ce7a4775d8 100644 --- a/pkg/upgrade/templates/v1/jiva_target.go +++ b/pkg/upgrade/templates/v1/jiva_target.go @@ -19,31 +19,32 @@ package templates var ( // JivaTargetPatch is generic template for target patch JivaTargetPatch = `{ - "metadata": { - "labels": { - "openebs.io/version": "{{.UpgradeVersion}}" - } - }, - "spec": { - "template": { - "metadata": { - "labels":{ - "openebs.io/version": "{{.UpgradeVersion}}" - } - }, - "spec": { - "containers": [ - { - "name": "{{.ControllerContainerName}}", - "image": "{{.ControllerImage}}:{{.ImageTag}}" - }, - { - "name": "maya-volume-exporter", - "image": "{{.MExporterImage}}:{{.ImageTag}}" - } - ] - } - } - } - }` + "metadata": { + "labels": { + "openebs.io/version": "{{.UpgradeVersion}}" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "labels": { + "openebs.io/version": "{{.UpgradeVersion}}" + } + }, + "spec": { + "containers": [ + { + "name": "{{.ControllerContainerName}}", + "image": "{{.ControllerImage}}:{{.ImageTag}}" + }, + { + "name": "maya-volume-exporter", + "image": "{{.MExporterImage}}:{{.ImageTag}}" + } + ] + } + } + } +}` ) diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index e70c2b5c1a..97280f52a3 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -29,6 +29,7 @@ import ( templates "github.com/openebs/maya/pkg/upgrade/templates/v1" errors "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" k8serror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -419,6 +420,43 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } +func scaleDownDeploy(name, namespace string) error { + klog.Infof("Scaling down target deploy %s in %s namespace", name, namespace) + deployObj, err := deployClient.WithNamespace(namespace).Get(name) + if err != nil { + return err + } + pvLabelKey := "openebs.io/persistent-volume" + pvName := deployObj.Labels[pvLabelKey] + controllerLabel := "openebs.io/controller=jiva-controller," + + pvLabelKey + "=" + pvName + var zero int32 + deployObj.Spec.Replicas = &zero + _, err = deployClient.WithNamespace(namespace).Update(deployObj) + if err != nil { + return err + } + podList := &corev1.PodList{} + for i := 0; i < 60; i++ { + podList, err = podClient.WithNamespace(namespace).List( + metav1.ListOptions{ + LabelSelector: controllerLabel, + }) + if err != nil { + return err + } + if len(podList.Items) > 0 { + time.Sleep(time.Second * 5) + } else { + break + } + } + if len(podList.Items) > 0 { + return errors.Errorf("target pod still present") + } + return nil +} + func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { var err, uerr error statusObj := utask.UpgradeDetailedStatuses{Step: utask.ReplicaUpgrade} @@ -429,6 +467,18 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { } statusObj.Phase = utask.StepErrored + + err = scaleDownDeploy(j.controllerObj.name, j.ns) + if err != nil { + statusObj.Message = "failed to scale down target depoyment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return errors.Wrap(err, "failed to scale down target depoyment") + } + // replica patch err = patchReplica(j.replicaObj, j.ns) if err != nil { From bda03f91096e905bbea8576db0f4d2eba0545dbb Mon Sep 17 00:00:00 2001 From: shubham Date: Wed, 11 Mar 2020 15:32:59 +0530 Subject: [PATCH 2/4] rename function Signed-off-by: shubham --- pkg/upgrade/upgrader/jiva_upgrade.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index 97280f52a3..fedb34d78b 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -420,7 +420,7 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } -func scaleDownDeploy(name, namespace string) error { +func scaleDownTargetDeploy(name, namespace string) error { klog.Infof("Scaling down target deploy %s in %s namespace", name, namespace) deployObj, err := deployClient.WithNamespace(namespace).Get(name) if err != nil { @@ -468,7 +468,7 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { statusObj.Phase = utask.StepErrored - err = scaleDownDeploy(j.controllerObj.name, j.ns) + err = scaleDownTargetDeploy(j.controllerObj.name, j.ns) if err != nil { statusObj.Message = "failed to scale down target depoyment" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) From b46a371270cfe08f911d78ef8ac667a92e8b34f8 Mon Sep 17 00:00:00 2001 From: shubham Date: Wed, 11 Mar 2020 15:57:01 +0530 Subject: [PATCH 3/4] addressed review comments Signed-off-by: shubham --- pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go | 2 +- pkg/upgrade/upgrader/jiva_upgrade.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go b/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go index a48e6b0fc3..6427224cca 100644 --- a/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go +++ b/pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go @@ -421,7 +421,7 @@ func (k *Kubeclient) Create(deployment *appsv1.Deployment) (*appsv1.Deployment, // Update updates a deployment in specified namespace in kubernetes cluster func (k *Kubeclient) Update(deployment *appsv1.Deployment) (*appsv1.Deployment, error) { - if debug.EI.IsDeploymentCreateErrorInjected() { + if debug.EI.IsDeploymentUpdateErrorInjected() { return nil, errors.New("Deployment update error via injection") } diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index fedb34d78b..0356cdeabf 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -437,6 +437,7 @@ func scaleDownTargetDeploy(name, namespace string) error { return err } podList := &corev1.PodList{} + // Wait for up to 5 minutes targe pod to go away. for i := 0; i < 60; i++ { podList, err = podClient.WithNamespace(namespace).List( metav1.ListOptions{ @@ -451,6 +452,7 @@ func scaleDownTargetDeploy(name, namespace string) error { break } } + // If pod is not deleted within 5 minutes return error. if len(podList.Items) > 0 { return errors.Errorf("target pod still present") } From 3a4a876cf4833ef8c05d11351831be6799b14285 Mon Sep 17 00:00:00 2001 From: shubham Date: Wed, 11 Mar 2020 16:06:48 +0530 Subject: [PATCH 4/4] fixed typo Signed-off-by: shubham --- pkg/upgrade/upgrader/jiva_upgrade.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index 0356cdeabf..d85a3f9c05 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -437,7 +437,7 @@ func scaleDownTargetDeploy(name, namespace string) error { return err } podList := &corev1.PodList{} - // Wait for up to 5 minutes targe pod to go away. + // Wait for up to 5 minutes for target pod to go away. for i := 0; i < 60; i++ { podList, err = podClient.WithNamespace(namespace).List( metav1.ListOptions{