refact(upgrade): scale down jiva target deploy before replica patch (#…

…1626)

Avoid the replica's from getting connected to the older version of
the controller and triggering a snapshot delete and so forth code. 

Signed-off-by: shubham <shubham.bajpai@mayadata.io>

pkg/kubernetes/deployment/appsv1/v1alpha1/kubernetes.go

@@ -17,9 +17,10 @@ package v1alpha1
 
 import (
 	"encoding/json"
-	"github.com/openebs/maya/pkg/debug"
 	"strings"
 
+	"github.com/openebs/maya/pkg/debug"
+
 	client "github.com/openebs/maya/pkg/kubernetes/client/v1alpha1"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
@@ -61,6 +62,14 @@ type createFn func(
 	deploy *appsv1.Deployment,
 ) (*appsv1.Deployment, error)
 
+// updateFn is a typed function that abstracts
+// updating a deployment instance in kubernetes cluster
+type updateFn func(
+	cli *kubernetes.Clientset,
+	namespace string,
+	deploy *appsv1.Deployment,
+) (*appsv1.Deployment, error)
+
 // deleteFn is a typed function that abstracts
 // deleting a deployment from kubernetes cluster
 type deleteFn func(
@@ -137,6 +146,17 @@ func defaultCreate(
 	return cli.AppsV1().Deployments(namespace).Create(deploy)
 }
 
+// defaultUpdate is the default implementation to update
+// a deployment instance in kubernetes cluster
+func defaultUpdate(
+	cli *kubernetes.Clientset,
+	namespace string,
+	deploy *appsv1.Deployment,
+) (*appsv1.Deployment, error) {
+
+	return cli.AppsV1().Deployments(namespace).Update(deploy)
+}
+
 // defaultDel is the default implementation to delete a
 // deployment instance in kubernetes cluster
 func defaultDel(
@@ -193,6 +213,7 @@ type Kubeclient struct {
 	get                 getFn
 	list                listFn
 	create              createFn
+	update              updateFn
 	del                 deleteFn
 	patch               patchFn
 	rolloutStatus       rolloutStatusFn
@@ -222,6 +243,9 @@ func (k *Kubeclient) withDefaults() {
 	if k.create == nil {
 		k.create = defaultCreate
 	}
+	if k.update == nil {
+		k.update = defaultUpdate
+	}
 	if k.del == nil {
 		k.del = defaultDel
 	}
@@ -394,6 +418,30 @@ func (k *Kubeclient) Create(deployment *appsv1.Deployment) (*appsv1.Deployment,
 	return k.create(cli, k.namespace, deployment)
 }
 
+// Update updates a deployment in specified namespace in kubernetes cluster
+func (k *Kubeclient) Update(deployment *appsv1.Deployment) (*appsv1.Deployment, error) {
+
+	if debug.EI.IsDeploymentUpdateErrorInjected() {
+		return nil, errors.New("Deployment update error via injection")
+	}
+
+	if deployment == nil {
+		return nil, errors.New("failed to update deployment: nil deployment object")
+	}
+
+	cli, err := k.getClientOrCached()
+	if err != nil {
+		return nil, errors.Wrapf(
+			err,
+			"failed to update deployment {%s} in namespace {%s}",
+			deployment.Name,
+			deployment.Namespace,
+		)
+	}
+
+	return k.update(cli, k.namespace, deployment)
+}
+
 // RolloutStatusf returns deployment's rollout status for given name
 // in raw bytes
 func (k *Kubeclient) RolloutStatusf(name string) (op []byte, err error) {

pkg/upgrade/templates/v1/jiva_target.go

@@ -19,31 +19,32 @@ package templates
 var (
 	// JivaTargetPatch is generic template for target patch
 	JivaTargetPatch = `{
-		"metadata": {
-		   "labels": {
-			 "openebs.io/version": "{{.UpgradeVersion}}"
-		   }
-		},
-		"spec": {
-		   "template": {
-			  "metadata": {
-				 "labels":{
-					"openebs.io/version": "{{.UpgradeVersion}}"
-				 }
-			  },
-			 "spec": {
-			   "containers": [
-				 {
-					"name": "{{.ControllerContainerName}}",
-					"image": "{{.ControllerImage}}:{{.ImageTag}}"
-				 },
-				 {
-					"name": "maya-volume-exporter",
-					"image": "{{.MExporterImage}}:{{.ImageTag}}"
-				 }
-			   ]
-			 }
-		   }
-		}
-	  }`
+  "metadata": {
+    "labels": {
+      "openebs.io/version": "{{.UpgradeVersion}}"
+    }
+  },
+  "spec": {
+    "replicas": 1,
+    "template": {
+      "metadata": {
+        "labels": {
+          "openebs.io/version": "{{.UpgradeVersion}}"
+        }
+      },
+      "spec": {
+        "containers": [
+          {
+            "name": "{{.ControllerContainerName}}",
+            "image": "{{.ControllerImage}}:{{.ImageTag}}"
+          },
+          {
+            "name": "maya-volume-exporter",
+            "image": "{{.MExporterImage}}:{{.ImageTag}}"
+          }
+        ]
+      }
+    }
+  }
+}`
 )

pkg/upgrade/upgrader/jiva_upgrade.go

@@ -29,6 +29,7 @@ import (
 	templates "github.com/openebs/maya/pkg/upgrade/templates/v1"
 	errors "github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	types "k8s.io/apimachinery/pkg/types"
@@ -419,6 +420,45 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error {
 	return nil
 }
 
+func scaleDownTargetDeploy(name, namespace string) error {
+	klog.Infof("Scaling down target deploy %s in %s namespace", name, namespace)
+	deployObj, err := deployClient.WithNamespace(namespace).Get(name)
+	if err != nil {
+		return err
+	}
+	pvLabelKey := "openebs.io/persistent-volume"
+	pvName := deployObj.Labels[pvLabelKey]
+	controllerLabel := "openebs.io/controller=jiva-controller," +
+		pvLabelKey + "=" + pvName
+	var zero int32
+	deployObj.Spec.Replicas = &zero
+	_, err = deployClient.WithNamespace(namespace).Update(deployObj)
+	if err != nil {
+		return err
+	}
+	podList := &corev1.PodList{}
+	// Wait for up to 5 minutes for target pod to go away.
+	for i := 0; i < 60; i++ {
+		podList, err = podClient.WithNamespace(namespace).List(
+			metav1.ListOptions{
+				LabelSelector: controllerLabel,
+			})
+		if err != nil {
+			return err
+		}
+		if len(podList.Items) > 0 {
+			time.Sleep(time.Second * 5)
+		} else {
+			break
+		}
+	}
+	// If pod is not deleted within 5 minutes return error.
+	if len(podList.Items) > 0 {
+		return errors.Errorf("target pod still present")
+	}
+	return nil
+}
+
 func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error {
 	var err, uerr error
 	statusObj := utask.UpgradeDetailedStatuses{Step: utask.ReplicaUpgrade}
@@ -429,6 +469,18 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error {
 	}
 
 	statusObj.Phase = utask.StepErrored
+
+	err = scaleDownTargetDeploy(j.controllerObj.name, j.ns)
+	if err != nil {
+		statusObj.Message = "failed to scale down target depoyment"
+		statusObj.Reason = strings.Replace(err.Error(), ":", "", -1)
+		j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace)
+		if uerr != nil && isENVPresent {
+			return uerr
+		}
+		return errors.Wrap(err, "failed to scale down target depoyment")
+	}
+
 	// replica patch
 	err = patchReplica(j.replicaObj, j.ns)
 	if err != nil {