From c85749cd1fb853ca22b48dbcfdba2d398915930d Mon Sep 17 00:00:00 2001 From: shubham Date: Thu, 2 Apr 2020 17:28:05 +0530 Subject: [PATCH 1/5] feat(upgrade): add support to migrate old jiva volumes Signed-off-by: shubham --- pkg/upgrade/upgrader/helper.go | 67 ++++++ pkg/upgrade/upgrader/jiva_upgrade.go | 330 ++++++++++++++++++++------- 2 files changed, 311 insertions(+), 86 deletions(-) diff --git a/pkg/upgrade/upgrader/helper.go b/pkg/upgrade/upgrader/helper.go index 19bbaf4b5f..3d2c909fbb 100644 --- a/pkg/upgrade/upgrader/helper.go +++ b/pkg/upgrade/upgrader/helper.go @@ -23,11 +23,13 @@ import ( apis "github.com/openebs/maya/pkg/apis/openebs.io/upgrade/v1alpha1" deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" + pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1" templates "github.com/openebs/maya/pkg/upgrade/templates/v1" utask "github.com/openebs/maya/pkg/upgrade/v1alpha2" retry "github.com/openebs/maya/pkg/util/retry" errors "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" k8serror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -318,3 +320,68 @@ func buildUpgradeTask(kind, name, openebsNamespace string) *apis.UpgradeTask { } return utaskObj } + +func scaleDeploy(name, namespace, label string, rc int) error { + klog.Infof("Scaling deploy %s in %s namespace to %d", name, namespace, rc) + deployObj, err := deployClient.WithNamespace(namespace).Get(name) + if err != nil { + return err + } + replicas := int32(rc) + deployObj.Spec.Replicas = &replicas + _, err = deployClient.WithNamespace(namespace).Update(deployObj) + if err != nil { + return err + } + podList := &corev1.PodList{} + // Wait for up to 5 minutes for deployment pods to reach desired replicaCount. + for i := 0; i < 60; i++ { + podList, err := podClient.WithNamespace(namespace).List( + metav1.ListOptions{ + LabelSelector: label, + }) + if err != nil { + return err + } + if len(podList.Items) != rc { + time.Sleep(time.Second * 5) + } else { + break + } + } + // If number pods is not reached within 5 minutes return error. + if len(podList.Items) != rc { + return errors.Errorf("expected pods: %d, found: %d", rc, len(podList.Items)) + } + return nil +} + +func waitUntilPodsAreRunning(namespace, label string, rc int) error { + pods := &corev1.PodList{} + count := 0 + var err error + // Wait for up to 5 minutes for deployment pods to reach desired replicaCount. + for i := 0; i < 60; i++ { + // GetPodRunningCount gives number of pods running currently + pods, err = podClient. + WithNamespace(namespace). + List(metav1.ListOptions{LabelSelector: label}) + if err != nil { + return err + } + count = pod. + ListBuilderForAPIList(pods). + WithFilter(pod.IsRunning()). + List(). + Len() + if count != rc { + time.Sleep(time.Second * 5) + } else { + break + } + } + if count != rc { + return errors.Errorf("expected running pods: %d, found: %d", rc, count) + } + return nil +} diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index 1635643dab..f281ff5ba4 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -25,12 +25,10 @@ import ( "time" utask "github.com/openebs/maya/pkg/apis/openebs.io/upgrade/v1alpha1" - apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" jivaClient "github.com/openebs/maya/pkg/client/jiva" templates "github.com/openebs/maya/pkg/upgrade/templates/v1" errors "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" k8serror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -411,6 +409,20 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return err } + if currentVersion < "1.9.0" { + klog.Infof("Scaling down old deployments") + err = j.preMigration(openebsNamespace) + if err != nil { + statusObj.Message = "failed to get scale down old deployments" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err + } + } + statusObj.Phase = utask.StepCompleted statusObj.Message = "Pre-upgrade steps were successful" statusObj.Reason = "" @@ -421,44 +433,6 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } -func scaleTargetDeploy(name, namespace string, rc int) error { - klog.Infof("Scaling down target deploy %s in %s namespace", name, namespace) - deployObj, err := deployClient.WithNamespace(namespace).Get(name) - if err != nil { - return err - } - pvName := deployObj.Labels[string(apis.PersistentVolumeCPK)] - controllerLabel := "openebs.io/controller=jiva-controller," + - string(apis.PersistentVolumeCPK) + "=" + pvName - replicas := int32(rc) - deployObj.Spec.Replicas = &replicas - _, err = deployClient.WithNamespace(namespace).Update(deployObj) - if err != nil { - return err - } - podList := &corev1.PodList{} - // Wait for up to 5 minutes for target pod to go away. - for i := 0; i < 60; i++ { - podList, err = podClient.WithNamespace(namespace).List( - metav1.ListOptions{ - LabelSelector: controllerLabel, - }) - if err != nil { - return err - } - if len(podList.Items) != rc { - time.Sleep(time.Second * 5) - } else { - break - } - } - // If pod is not deleted within 5 minutes return error. - if len(podList.Items) != rc { - return errors.Errorf("expected pods: %d, found: %d", rc, len(podList.Items)) - } - return nil -} - func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { var err, uerr error statusObj := utask.UpgradeDetailedStatuses{Step: utask.ReplicaUpgrade} @@ -470,38 +444,52 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { statusObj.Phase = utask.StepErrored - // Scaling down controller deployment before patching replica deployment - // if the replica is not upgraded already. - if j.replicaObj.version == currentVersion { - err = scaleTargetDeploy(j.controllerObj.name, j.ns, 0) + if currentVersion < "1.9.0" { + err = j.migrateReplica(openebsNamespace) if err != nil { - statusObj.Message = "failed to scale down target depoyment" + statusObj.Message = "failed to migrate replica to openebs namespace" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) if uerr != nil && isENVPresent { return uerr } - return errors.Wrap(err, "failed to scale down target depoyment") + return errors.Wrap(err, "failed to migrate replica to openebs namespace") } - } + } else { - // replica patch - err = patchReplica(j.replicaObj, j.ns) - if err != nil { - // If patching of replica fails the controller needs to be reverted - // as the upgrade will not proceed with controller patch. - scaleErr := scaleTargetDeploy(j.controllerObj.name, j.ns, 1) - if scaleErr != nil { - klog.Infof("failed to scale up controller delpoyment. Please scale up deployment "+ - "%s in %s namespace to 1 manually.", j.controllerObj.name, j.ns) - } - statusObj.Message = "failed to patch replica depoyment" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr + // Scaling down controller deployment before patching replica deployment + // if the replica is not upgraded already. + if j.replicaObj.version == currentVersion { + err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) + if err != nil { + statusObj.Message = "failed to scale down target depoyment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return errors.Wrap(err, "failed to scale down target depoyment") + } + } + + // replica patch + err = patchReplica(j.replicaObj, j.ns) + if err != nil { + // If patching of replica fails the controller needs to be reverted + // as the upgrade will not proceed with controller patch. + scaleErr := scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 1) + if scaleErr != nil { + klog.Infof("failed to scale up controller delpoyment. Please scale up deployment "+ + "%s in %s namespace to 1 manually.", j.controllerObj.name, j.ns) + } + statusObj.Message = "failed to patch replica depoyment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err } - return err } statusObj.Phase = utask.StepCompleted @@ -524,29 +512,43 @@ func (j *jivaVolumeOptions) targetUpgrade(pvName, openebsNamespace string) error } statusObj.Phase = utask.StepErrored - // controller patch - err = patchController(j.controllerObj, j.ns) - if err != nil { - statusObj.Message = "failed to patch target depoyment" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr + + if currentVersion < "1.9.0" && j.ns != openebsNamespace { + err = j.migrateTarget(openebsNamespace) + if err != nil { + statusObj.Message = "failed to migrate target to openebs namespace" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err } - return err - } - pvLabel := "openebs.io/persistent-volume=" + pvName - serviceLabel := "openebs.io/controller-service=jiva-controller-svc," + pvLabel + } else { + // controller patch + err = patchController(j.controllerObj, j.ns) + if err != nil { + statusObj.Message = "failed to patch target depoyment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err + } + pvLabel := "openebs.io/persistent-volume=" + pvName + serviceLabel := "openebs.io/controller-service=jiva-controller-svc," + pvLabel - err = patchService(serviceLabel, j.ns) - if err != nil { - statusObj.Message = "failed to patch target service" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr + err = patchService(serviceLabel, j.ns) + if err != nil { + statusObj.Message = "failed to patch target service" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err } - return err } statusObj.Phase = utask.StepCompleted @@ -560,6 +562,9 @@ func (j *jivaVolumeOptions) targetUpgrade(pvName, openebsNamespace string) error } func (j *jivaVolumeOptions) verify(pvLabel, openebsNamespace string) error { + // after the new ctrl and svc it takes few seconds for the + // tcp connection to start + time.Sleep(10 * time.Second) var err, uerr error statusObj := utask.UpgradeDetailedStatuses{Step: utask.Verify} statusObj.Phase = utask.StepWaiting @@ -570,7 +575,7 @@ func (j *jivaVolumeOptions) verify(pvLabel, openebsNamespace string) error { statusObj.Phase = utask.StepErrored // Verify synced replicas - err = validateSync(pvLabel, j.ns) + err = validateSync(pvLabel, openebsNamespace) if err != nil { statusObj.Message = "failed to verify synced replicas. Please check it manually using the steps mentioned in https://docs.openebs.io/docs/next/mayactl.html" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) @@ -595,13 +600,24 @@ func (j *jivaVolumeOptions) verify(pvLabel, openebsNamespace string) error { return nil } +var ( + ctrlDeployLabel, replicaDeployLabel, ctrlSVCLabel string +) + func jivaUpgrade(pvName, openebsNamespace string, utaskObj *utask.UpgradeTask) (*utask.UpgradeTask, error) { var ( - pvLabel = "openebs.io/persistent-volume=" + pvName - err error + pvLabel = "openebs.io/persistent-volume=" + pvName + ctrlLabel = "openebs.io/controller=jiva-controller," + replicaLabel = "openebs.io/replica=jiva-replica," + svcLabel = "openebs.io/controller-service=jiva-controller-svc," + err error ) + ctrlDeployLabel = ctrlLabel + pvLabel + replicaDeployLabel = replicaLabel + pvLabel + ctrlSVCLabel = svcLabel + pvLabel + options := &jivaVolumeOptions{} options.utaskObj = utaskObj @@ -630,6 +646,148 @@ func jivaUpgrade(pvName, openebsNamespace string, utaskObj *utask.UpgradeTask) ( return options.utaskObj, err } + err = options.cleanup(openebsNamespace) + if err != nil { + return options.utaskObj, err + } + klog.Info("Upgrade Successful for", pvName) return options.utaskObj, nil } + +func (j *jivaVolumeOptions) cleanup(openebsNamespace string) error { + var err error + if j.replicaObj.version == currentVersion { + err = deployClient.WithNamespace(j.ns).Delete(j.replicaObj.name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } + if j.controllerObj.version == currentVersion && j.ns != openebsNamespace { + err = deployClient.WithNamespace(j.ns).Delete(j.controllerObj.name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (j *jivaVolumeOptions) preMigration(openebsNamespace string) error { + var err error + if j.replicaObj.version == currentVersion { + err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) + if err != nil { + return err + } + } + if j.controllerObj.version == currentVersion { + err = scaleDeploy(j.replicaObj.name, j.ns, replicaDeployLabel, 0) + if err != nil { + return err + } + } + + return nil +} + +func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { + if j.replicaObj.version == currentVersion { + klog.Infof("splitting replica deployment") + deployObj, err := deployClient.WithNamespace(j.ns).Get(j.replicaObj.name) + if err != nil { + return err + } + nodeNames := deployObj.Spec.Template.Spec.Affinity.NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values + var one int32 = 1 + for i, node := range nodeNames { + replicaDeploy := deployObj.DeepCopy() + replicaDeploy.Name = replicaDeploy.Name + "-" + strconv.Itoa(i+1) + replicaDeploy.Namespace = openebsNamespace + replicaDeploy.ResourceVersion = "" + replicaDeploy.Spec.Replicas = &one + // updating the replica deployments image and version before + // creating them in openebs namespace. + lastIndex := strings.LastIndex(replicaDeploy.Spec.Template.Spec.Containers[0].Image, ":") + replicaDeploy.Spec.Template.Spec.Containers[0].Image = replicaDeploy.Spec. + Template.Spec.Containers[0].Image[:lastIndex+1] + imageTag + replicaDeploy.Labels["openebs.io/version"] = upgradeVersion + replicaDeploy.Spec.Template.Spec.Affinity.NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values = []string{node} + klog.Infof("creating replica deployment %s in %s namespace", replicaDeploy.Name, openebsNamespace) + replicaDeploy, err := deployClient.WithNamespace(openebsNamespace).Create(replicaDeploy) + if err != nil { + return err + } + err = waitUntilPodsAreRunning(openebsNamespace, replicaDeployLabel, i+1) + if err != nil { + return err + } + } + } + return nil +} + +func (j *jivaVolumeOptions) migrateTarget(openebsNamespace string) error { + if j.controllerObj.version == currentVersion { + deployObj, err := deployClient.WithNamespace(j.ns).Get(j.controllerObj.name) + if err != nil { + return err + } + var one int32 = 1 + deployObj.Spec.Replicas = &one + deployObj.Namespace = openebsNamespace + deployObj.ResourceVersion = "" + // updating the controller deploymentsimage and version before + // creating them in openebs namespace. + lastIndex := strings.LastIndex(deployObj.Spec.Template.Spec.Containers[0].Image, ":") + deployObj.Spec.Template.Spec.Containers[0].Image = deployObj.Spec. + Template.Spec.Containers[0].Image[:lastIndex+1] + imageTag + lastIndex = strings.LastIndex(deployObj.Spec.Template.Spec.Containers[1].Image, ":") + deployObj.Spec.Template.Spec.Containers[1].Image = deployObj.Spec. + Template.Spec.Containers[1].Image[:lastIndex+1] + imageTag + // if target-affinity is set for the pvc them openebs namespace + // needs to be added as a bug fix. + if deployObj.Spec.Template.Spec.Affinity != nil { + if deployObj.Spec.Template.Spec.Affinity.PodAffinity. + RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector. + MatchExpressions[0].Key == "openebs.io/target-affinity" { + deployObj.Spec.Template.Spec.Affinity.PodAffinity. + RequiredDuringSchedulingIgnoredDuringExecution[0]. + Namespaces = []string{j.ns} + } + } + + deployObj.Labels["openebs.io/version"] = upgradeVersion + klog.Infof("creating controller deployment %s in %s namespace", deployObj.Name, openebsNamespace) + deployObj, err = deployClient.WithNamespace(openebsNamespace).Create(deployObj) + if err != nil { + return err + } + err = waitUntilPodsAreRunning(openebsNamespace, ctrlDeployLabel, 1) + if err != nil { + return err + } + svcList, err := serviceClient.WithNamespace(j.ns).List(metav1.ListOptions{ + LabelSelector: ctrlSVCLabel, + }) + if err != nil { + return err + } + svcObj := &svcList.Items[0] + svcObj.Namespace = openebsNamespace + svcObj.ResourceVersion = "" + svcObj.Labels["openebs.io/version"] = upgradeVersion + klog.Infof("removing controller service %s in %s namespace", svcObj.Name, j.ns) + err = serviceClient.WithNamespace(j.ns).Delete(svcObj.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + klog.Infof("creating controller service %s in %s namespace", svcObj.Name, openebsNamespace) + svcObj, err = serviceClient.WithNamespace(openebsNamespace).Create(svcObj) + if err != nil { + return err + } + } + return nil +} From 56499e9a86cdfd32310c5f7b876e2ccfe67f6a7a Mon Sep 17 00:00:00 2001 From: shubham Date: Fri, 3 Apr 2020 23:12:03 +0530 Subject: [PATCH 2/5] - make the job more fault resistant - segregate migration from upgarde Signed-off-by: shubham --- pkg/upgrade/templates/v1/jiva_replica.go | 1 + pkg/upgrade/upgrader/jiva_upgrade.go | 420 ++++++++++++++--------- 2 files changed, 253 insertions(+), 168 deletions(-) diff --git a/pkg/upgrade/templates/v1/jiva_replica.go b/pkg/upgrade/templates/v1/jiva_replica.go index 6441891a61..1fd88b43fd 100644 --- a/pkg/upgrade/templates/v1/jiva_replica.go +++ b/pkg/upgrade/templates/v1/jiva_replica.go @@ -27,6 +27,7 @@ var ( } }, "spec": { + "replicas": 1, "selector": { "matchLabels":{ "openebs.io/persistent-volume": "{{.PVName}}", diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index f281ff5ba4..18a20d3984 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -26,12 +26,15 @@ import ( utask "github.com/openebs/maya/pkg/apis/openebs.io/upgrade/v1alpha1" jivaClient "github.com/openebs/maya/pkg/client/jiva" + sc "github.com/openebs/maya/pkg/kubernetes/storageclass/v1alpha1" templates "github.com/openebs/maya/pkg/upgrade/templates/v1" errors "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" k8serror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/klog" // auth plugins @@ -49,6 +52,7 @@ type controllerPatchDetails struct { type replicaDetails struct { patchDetails *replicaPatchDetails version, name string + replicas map[string]string } type controllerDetails struct { @@ -115,35 +119,81 @@ func getControllerPatchDetails(d *appsv1.Deployment) ( return patchDetails, nil } -func getReplica(replicaLabel, namespace string) (*replicaDetails, error) { - replicaObj := &replicaDetails{} - deployObj, err := getDeployment(replicaLabel, namespace) - if err != nil { - return nil, errors.Wrapf(err, "failed to get replica deployment") +func getReplica(pvName, replicaLabel, namespace, openebsNamespace string) (*replicaDetails, error) { + replicaObj := &replicaDetails{ + replicas: map[string]string{}, } - if deployObj.Name == "" { - return nil, errors.Errorf("missing deployment name for replica") + // check if old replica is present for currentVersion < 1.9.0 + // if present then migration is not complete and store the old + // replica details + if currentVersion < "1.9.0" { + deployObj, err := deployClient.WithNamespace(namespace).Get(pvName + "-rep") + if err != nil && !k8serror.IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to get replica deployment") + } + if err == nil { + klog.Info("getting old deployment") + if deployObj.Name == "" { + return nil, errors.Errorf("missing deployment name for replica") + } + replicaObj.name = deployObj.Name + version, err := getOpenEBSVersion(deployObj) + if err != nil { + return nil, err + } + if (version != currentVersion) && (version != upgradeVersion) { + return nil, errors.Errorf( + "replica version %s is neither %s nor %s\n", + version, + currentVersion, + upgradeVersion, + ) + } + replicaObj.version = version + patchDetails, err := getReplicaPatchDetails(deployObj) + if err != nil { + return nil, err + } + replicaObj.patchDetails = patchDetails + replicaObj.patchDetails.UpgradeVersion = upgradeVersion + replicaObj.patchDetails.PVName = pvName + } } - replicaObj.name = deployObj.Name - version, err := getOpenEBSVersion(deployObj) + // + replicaList, err := deployClient.WithNamespace(openebsNamespace).List(&metav1.ListOptions{ + LabelSelector: replicaLabel, + }) if err != nil { return nil, err } - if (version != currentVersion) && (version != upgradeVersion) { - return nil, errors.Errorf( - "replica version %s is neither %s nor %s\n", - version, - currentVersion, - upgradeVersion, - ) - } - replicaObj.version = version - patchDetails, err := getReplicaPatchDetails(deployObj) - if err != nil { - return nil, err + for _, replica := range replicaList.Items { + // skip the old deployment as that will + // be removed and not patched + if replica.Name != pvName+"-rep" { + deployObj := &replica + version, err := getOpenEBSVersion(deployObj) + if err != nil { + return nil, err + } + if (version != currentVersion) && (version != upgradeVersion) { + return nil, errors.Errorf( + "replica %s version %s is neither %s nor %s\n", + deployObj.Name, + version, + currentVersion, + upgradeVersion, + ) + } + replicaObj.replicas[deployObj.Name] = version + patchDetails, err := getReplicaPatchDetails(deployObj) + if err != nil { + return nil, err + } + replicaObj.patchDetails = patchDetails + replicaObj.patchDetails.UpgradeVersion = upgradeVersion + replicaObj.patchDetails.PVName = pvName + } } - replicaObj.patchDetails = patchDetails - replicaObj.patchDetails.UpgradeVersion = upgradeVersion return replicaObj, nil } @@ -179,8 +229,8 @@ func getController(controllerLabel, namespace string) (*controllerDetails, error return controllerObj, nil } -func patchReplica(replicaObj *replicaDetails, namespace string) error { - if replicaObj.version == currentVersion { +func patchReplica(name, namespace string, replicaObj *replicaDetails) error { + if replicaObj.replicas[name] == currentVersion { tmpl, err := template.New("replicaPatch"). Parse(templates.JivaReplicaPatch) if err != nil { @@ -193,17 +243,17 @@ func patchReplica(replicaObj *replicaDetails, namespace string) error { replicaPatch := buffer.String() buffer.Reset() err = patchDelpoyment( - replicaObj.name, + name, namespace, types.StrategicMergePatchType, []byte(replicaPatch), ) if err != nil { - return errors.Wrapf(err, "failed to patch replica deployment") + return errors.Wrapf(err, "failed to patch replica deployment %s", name) } - klog.Infof("%s patched", replicaObj.name) + klog.Infof("%s patched", name) } else { - klog.Infof("replica deployment already in %s version", upgradeVersion) + klog.Infof("replica deployment %s already in %s version", name, upgradeVersion) } return nil } @@ -240,31 +290,30 @@ func patchController(controllerObj *controllerDetails, namespace string) error { func getPVCDeploymentsNamespace( pvName, pvLabel, - openebsNamespace string) (ns string, err error) { + openebsNamespace string) (string, error) { pvObj, err := pvClient.Get(pvName, metav1.GetOptions{}) if err != nil { return "", err } - // verifying whether the pvc is deployed with DeployInOpenebsNamespace cas config - deployList, err := deployClient.WithNamespace(openebsNamespace).List( + if pvObj.Spec.ClaimRef.Namespace == "" { + return "", errors.Errorf("namespace missing for pv %s", pvName) + } + ns := pvObj.Spec.ClaimRef.Namespace + // check for pv deployments in pv refclaim namespace + deployList, err := deployClient.WithNamespace(ns).List( &metav1.ListOptions{ LabelSelector: pvLabel, }) if err != nil { return "", err } - // check whether pvc pods are openebs namespace or not + // check whether pvc pods are pvc namespace or not if len(deployList.Items) > 0 { - ns = openebsNamespace return ns, nil } - // if pvc pods are not in openebs namespace take the namespace of pvc - if pvObj.Spec.ClaimRef.Namespace == "" { - return "", errors.Errorf("namespace missing for pv %s", pvName) - } - ns = pvObj.Spec.ClaimRef.Namespace - // check for pv deployments in pv refclaim namespace - deployList, err = deployClient.WithNamespace(ns).List( + // if pvc pods are not in pvc namespace + // verifying whether the pvc is deployed with DeployInOpenebsNamespace cas config + deployList, err = deployClient.WithNamespace(openebsNamespace).List( &metav1.ListOptions{ LabelSelector: pvLabel, }) @@ -279,7 +328,7 @@ func getPVCDeploymentsNamespace( ns, ) } - return ns, nil + return openebsNamespace, nil } func getReplicationFactor(ctrlLabel, namespace string) (int, error) { @@ -385,7 +434,7 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { } // fetching replica deployment details - j.replicaObj, err = getReplica(replicaLabel, j.ns) + j.replicaObj, err = getReplica(pvName, replicaLabel, j.ns, openebsNamespace) if err != nil { statusObj.Message = "failed to get replica details" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) @@ -395,7 +444,6 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { } return err } - j.replicaObj.patchDetails.PVName = pvName // fetching controller deployment details j.controllerObj, err = getController(controllerLabel, j.ns) @@ -410,7 +458,6 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { } if currentVersion < "1.9.0" { - klog.Infof("Scaling down old deployments") err = j.preMigration(openebsNamespace) if err != nil { statusObj.Message = "failed to get scale down old deployments" @@ -444,7 +491,7 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { statusObj.Phase = utask.StepErrored - if currentVersion < "1.9.0" { + if currentVersion < "1.9.0" && j.replicaObj.name != "" { err = j.migrateReplica(openebsNamespace) if err != nil { statusObj.Message = "failed to migrate replica to openebs namespace" @@ -455,34 +502,14 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { } return errors.Wrap(err, "failed to migrate replica to openebs namespace") } - } else { - - // Scaling down controller deployment before patching replica deployment - // if the replica is not upgraded already. - if j.replicaObj.version == currentVersion { - err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) - if err != nil { - statusObj.Message = "failed to scale down target depoyment" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return errors.Wrap(err, "failed to scale down target depoyment") - } - } + } + for name := range j.replicaObj.replicas { // replica patch - err = patchReplica(j.replicaObj, j.ns) + klog.Info("patching replica deployments") + err = patchReplica(name, openebsNamespace, j.replicaObj) if err != nil { - // If patching of replica fails the controller needs to be reverted - // as the upgrade will not proceed with controller patch. - scaleErr := scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 1) - if scaleErr != nil { - klog.Infof("failed to scale up controller delpoyment. Please scale up deployment "+ - "%s in %s namespace to 1 manually.", j.controllerObj.name, j.ns) - } - statusObj.Message = "failed to patch replica depoyment" + statusObj.Message = "failed to patch replica depoyment " + name statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) if uerr != nil && isENVPresent { @@ -514,7 +541,7 @@ func (j *jivaVolumeOptions) targetUpgrade(pvName, openebsNamespace string) error statusObj.Phase = utask.StepErrored if currentVersion < "1.9.0" && j.ns != openebsNamespace { - err = j.migrateTarget(openebsNamespace) + err = j.migrateTarget(pvName, openebsNamespace) if err != nil { statusObj.Message = "failed to migrate target to openebs namespace" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) @@ -524,31 +551,30 @@ func (j *jivaVolumeOptions) targetUpgrade(pvName, openebsNamespace string) error } return err } - } else { - // controller patch - err = patchController(j.controllerObj, j.ns) - if err != nil { - statusObj.Message = "failed to patch target depoyment" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return err + } + // controller patch + err = patchController(j.controllerObj, openebsNamespace) + if err != nil { + statusObj.Message = "failed to patch target depoyment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr } - pvLabel := "openebs.io/persistent-volume=" + pvName - serviceLabel := "openebs.io/controller-service=jiva-controller-svc," + pvLabel + return err + } + pvLabel := "openebs.io/persistent-volume=" + pvName + serviceLabel := "openebs.io/controller-service=jiva-controller-svc," + pvLabel - err = patchService(serviceLabel, j.ns) - if err != nil { - statusObj.Message = "failed to patch target service" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return err + err = patchService(serviceLabel, openebsNamespace) + if err != nil { + statusObj.Message = "failed to patch target service" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr } + return err } statusObj.Phase = utask.StepCompleted @@ -656,96 +682,77 @@ func jivaUpgrade(pvName, openebsNamespace string, utaskObj *utask.UpgradeTask) ( } func (j *jivaVolumeOptions) cleanup(openebsNamespace string) error { - var err error - if j.replicaObj.version == currentVersion { - err = deployClient.WithNamespace(j.ns).Delete(j.replicaObj.name, &metav1.DeleteOptions{}) - if err != nil { - return err - } - } - if j.controllerObj.version == currentVersion && j.ns != openebsNamespace { - err = deployClient.WithNamespace(j.ns).Delete(j.controllerObj.name, &metav1.DeleteOptions{}) - if err != nil { - return err - } - } return nil } func (j *jivaVolumeOptions) preMigration(openebsNamespace string) error { - var err error - if j.replicaObj.version == currentVersion { - err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) - if err != nil { - return err - } - } - if j.controllerObj.version == currentVersion { - err = scaleDeploy(j.replicaObj.name, j.ns, replicaDeployLabel, 0) - if err != nil { - return err - } - } - return nil } func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { - if j.replicaObj.version == currentVersion { + // get the separate replica deployments in openebs namespace + deployList, err := deployClient.WithNamespace(openebsNamespace).List(&metav1.ListOptions{ + LabelSelector: replicaDeployLabel, + }) + if err != nil { + return err + } + // get the old replica deployment by name + oldDeployObj, err := deployClient.WithNamespace(j.ns).Get(j.replicaObj.name) + if err != nil { + return err + } + replicaCount := int(*oldDeployObj.Spec.Replicas) + replicasLeft := len(deployList.Items) + // if the volume was deployed in openebs namespace while provisioning + // as the old deployment also has the same label + if j.ns == openebsNamespace { + replicasLeft = replicasLeft - 1 + } + if len(deployList.Items) != replicaCount { klog.Infof("splitting replica deployment") - deployObj, err := deployClient.WithNamespace(j.ns).Get(j.replicaObj.name) - if err != nil { - return err - } - nodeNames := deployObj.Spec.Template.Spec.Affinity.NodeAffinity. + nodeNames := oldDeployObj.Spec.Template.Spec.Affinity.NodeAffinity. RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values - var one int32 = 1 - for i, node := range nodeNames { - replicaDeploy := deployObj.DeepCopy() + var zero int32 + for i := replicasLeft; i < replicaCount; i++ { + replicaDeploy := oldDeployObj.DeepCopy() replicaDeploy.Name = replicaDeploy.Name + "-" + strconv.Itoa(i+1) replicaDeploy.Namespace = openebsNamespace replicaDeploy.ResourceVersion = "" - replicaDeploy.Spec.Replicas = &one - // updating the replica deployments image and version before - // creating them in openebs namespace. - lastIndex := strings.LastIndex(replicaDeploy.Spec.Template.Spec.Containers[0].Image, ":") - replicaDeploy.Spec.Template.Spec.Containers[0].Image = replicaDeploy.Spec. - Template.Spec.Containers[0].Image[:lastIndex+1] + imageTag - replicaDeploy.Labels["openebs.io/version"] = upgradeVersion + replicaDeploy.Spec.Replicas = &zero replicaDeploy.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values = []string{node} + RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values = []string{nodeNames[i]} klog.Infof("creating replica deployment %s in %s namespace", replicaDeploy.Name, openebsNamespace) replicaDeploy, err := deployClient.WithNamespace(openebsNamespace).Create(replicaDeploy) if err != nil { return err } - err = waitUntilPodsAreRunning(openebsNamespace, replicaDeployLabel, i+1) - if err != nil { - return err - } + j.replicaObj.replicas[replicaDeploy.Name] = replicaDeploy.Labels["openebs.io/version"] } } + err = deployClient.WithNamespace(j.ns).Delete(j.replicaObj.name, &metav1.DeleteOptions{}) + if err != nil { + return err + } return nil } -func (j *jivaVolumeOptions) migrateTarget(openebsNamespace string) error { - if j.controllerObj.version == currentVersion { - deployObj, err := deployClient.WithNamespace(j.ns).Get(j.controllerObj.name) +func (j *jivaVolumeOptions) migrateTarget(pvName, openebsNamespace string) error { + // get the controller deployment in openebs namespace + deployObj, err := deployClient.WithNamespace(openebsNamespace).Get(j.controllerObj.name) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + // if the deployment is not found in openebs namespace migrate it + if k8serror.IsNotFound(err) { + var zero int32 + deployObj, err = deployClient.WithNamespace(j.ns).Get(j.controllerObj.name) if err != nil { return err } - var one int32 = 1 - deployObj.Spec.Replicas = &one deployObj.Namespace = openebsNamespace deployObj.ResourceVersion = "" - // updating the controller deploymentsimage and version before - // creating them in openebs namespace. - lastIndex := strings.LastIndex(deployObj.Spec.Template.Spec.Containers[0].Image, ":") - deployObj.Spec.Template.Spec.Containers[0].Image = deployObj.Spec. - Template.Spec.Containers[0].Image[:lastIndex+1] + imageTag - lastIndex = strings.LastIndex(deployObj.Spec.Template.Spec.Containers[1].Image, ":") - deployObj.Spec.Template.Spec.Containers[1].Image = deployObj.Spec. - Template.Spec.Containers[1].Image[:lastIndex+1] + imageTag + deployObj.Spec.Replicas = &zero // if target-affinity is set for the pvc them openebs namespace // needs to be added as a bug fix. if deployObj.Spec.Template.Spec.Affinity != nil { @@ -758,30 +765,45 @@ func (j *jivaVolumeOptions) migrateTarget(openebsNamespace string) error { } } - deployObj.Labels["openebs.io/version"] = upgradeVersion klog.Infof("creating controller deployment %s in %s namespace", deployObj.Name, openebsNamespace) deployObj, err = deployClient.WithNamespace(openebsNamespace).Create(deployObj) if err != nil { return err } - err = waitUntilPodsAreRunning(openebsNamespace, ctrlDeployLabel, 1) - if err != nil { - return err - } + } + + // get the controller service in openebs namespace + _, err = serviceClient.WithNamespace(openebsNamespace). + Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + // if the service is not found in openebs namespace migrate it + if k8serror.IsNotFound(err) { svcList, err := serviceClient.WithNamespace(j.ns).List(metav1.ListOptions{ LabelSelector: ctrlSVCLabel, }) if err != nil { return err } - svcObj := &svcList.Items[0] - svcObj.Namespace = openebsNamespace - svcObj.ResourceVersion = "" - svcObj.Labels["openebs.io/version"] = upgradeVersion - klog.Infof("removing controller service %s in %s namespace", svcObj.Name, j.ns) - err = serviceClient.WithNamespace(j.ns).Delete(svcObj.Name, &metav1.DeleteOptions{}) - if err != nil { - return err + svcObj := &corev1.Service{} + // if service is found in pvc namespace then recreate using it + // otherwise create the service using pv + if len(svcList.Items) != 0 { + svcObj = &svcList.Items[0] + svcObj.Namespace = openebsNamespace + svcObj.ResourceVersion = "" + klog.Infof("removing controller service %s in %s namespace", svcObj.Name, j.ns) + err = serviceClient.WithNamespace(j.ns).Delete(svcObj.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } else { + svcObj, err = getTargetSVC(pvName, openebsNamespace) + if err != nil { + return err + } + svcObj.Labels["openebs.io/cas-template-name"] = deployObj.Labels["openebs.io/cas-template-name"] } klog.Infof("creating controller service %s in %s namespace", svcObj.Name, openebsNamespace) svcObj, err = serviceClient.WithNamespace(openebsNamespace).Create(svcObj) @@ -789,5 +811,67 @@ func (j *jivaVolumeOptions) migrateTarget(openebsNamespace string) error { return err } } + err = deployClient.WithNamespace(j.ns).Delete(j.controllerObj.name, &metav1.DeleteOptions{}) + if err != nil && !k8serror.IsNotFound(err) { + return err + } return nil } + +func getTargetSVC(pvName, openebsNamespace string) (*corev1.Service, error) { + pvObj, err := pvClient.Get(pvName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + storageClass := pvObj.Spec.StorageClassName + storageClassObj, err := sc.NewKubeClient().Get(storageClass, metav1.GetOptions{}) + if err != nil { + return nil, err + } + svcObj := &corev1.Service{} + svcObj.ObjectMeta = metav1.ObjectMeta{ + Name: pvName + "-ctrl-svc", + Annotations: map[string]string{ + "openebs.io/storage-class-ref": `| + name: ` + storageClass + `\n` + + `resourceVersion: ` + storageClassObj.ResourceVersion, + }, + Labels: map[string]string{ + "openebs.io/storage-engine-type": "jiva", + "openebs.io/cas-type": "jiva", + "openebs.io/controller-service": "jiva-controller-svc", + "openebs.io/persistent-volume": pvName, + "openebs.io/persistent-volume-claim": pvObj.Spec.ClaimRef.Name, + "pvc": pvObj.Spec.ClaimRef.Name, + "openebs.io/version": currentVersion, + }, + } + svcObj.Spec = corev1.ServiceSpec{ + ClusterIP: strings.Split(pvObj.Spec.ISCSI.TargetPortal, ":")[0], + Ports: []corev1.ServicePort{ + corev1.ServicePort{ + Name: "iscsi", + Port: 3260, + Protocol: "TCP", + TargetPort: intstr.FromInt(3260), + }, + corev1.ServicePort{ + Name: "api", + Port: 9501, + Protocol: "TCP", + TargetPort: intstr.FromInt(9501), + }, + corev1.ServicePort{ + Name: "exporter", + Port: 9500, + Protocol: "TCP", + TargetPort: intstr.FromInt(9500), + }, + }, + Selector: map[string]string{ + "openebs.io/controller": "jiva-controller", + "openebs.io/persistent-volume": pvName, + }, + } + return svcObj, nil +} From b4de7739c59fd9567d1a4ce8326bae2ff872d39e Mon Sep 17 00:00:00 2001 From: shubham Date: Sun, 5 Apr 2020 12:15:04 +0530 Subject: [PATCH 3/5] move migration code together and cleanup of old resources is done after verifying upgrade Signed-off-by: shubham --- pkg/upgrade/upgrader/jiva_upgrade.go | 145 ++++++++++++++++----------- 1 file changed, 88 insertions(+), 57 deletions(-) diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index 18a20d3984..785cca7f28 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -26,7 +26,6 @@ import ( utask "github.com/openebs/maya/pkg/apis/openebs.io/upgrade/v1alpha1" jivaClient "github.com/openebs/maya/pkg/client/jiva" - sc "github.com/openebs/maya/pkg/kubernetes/storageclass/v1alpha1" templates "github.com/openebs/maya/pkg/upgrade/templates/v1" errors "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" @@ -458,9 +457,9 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { } if currentVersion < "1.9.0" { - err = j.preMigration(openebsNamespace) + err = j.migrate(pvName, openebsNamespace) if err != nil { - statusObj.Message = "failed to get scale down old deployments" + statusObj.Message = "failed to migrate deployments in openebes namespace" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) if uerr != nil && isENVPresent { @@ -480,7 +479,7 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } -func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { +func (j *jivaVolumeOptions) replicaUpgrade(pvName, openebsNamespace string) error { var err, uerr error statusObj := utask.UpgradeDetailedStatuses{Step: utask.ReplicaUpgrade} statusObj.Phase = utask.StepWaiting @@ -491,16 +490,36 @@ func (j *jivaVolumeOptions) replicaUpgrade(openebsNamespace string) error { statusObj.Phase = utask.StepErrored - if currentVersion < "1.9.0" && j.replicaObj.name != "" { - err = j.migrateReplica(openebsNamespace) + if currentVersion < "1.9.0" { + err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) + if err != nil { + statusObj.Message = "failed to get scale down target deployment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err + } + err = scaleDeploy(j.replicaObj.name, j.ns, replicaDeployLabel, 0) if err != nil { - statusObj.Message = "failed to migrate replica to openebs namespace" + statusObj.Message = "failed to get scale down replica deployment" statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) if uerr != nil && isENVPresent { return uerr } - return errors.Wrap(err, "failed to migrate replica to openebs namespace") + return err + } + err = j.migrateTargetSVC(pvName, openebsNamespace) + if err != nil { + statusObj.Message = "failed to get migrate target service" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err } } @@ -615,6 +634,18 @@ func (j *jivaVolumeOptions) verify(pvLabel, openebsNamespace string) error { } return err } + if currentVersion < "1.9.0" { + err = j.cleanup(openebsNamespace) + if err != nil { + statusObj.Message = "failed to clean up old replica deployemts" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr + } + return err + } + } statusObj.Phase = utask.StepCompleted statusObj.Message = "Replica sync was successful" @@ -655,7 +686,7 @@ func jivaUpgrade(pvName, openebsNamespace string, utaskObj *utask.UpgradeTask) ( } // ReplicaUpgrade - err = options.replicaUpgrade(openebsNamespace) + err = options.replicaUpgrade(pvName, openebsNamespace) if err != nil { return options.utaskObj, err } @@ -672,21 +703,36 @@ func jivaUpgrade(pvName, openebsNamespace string, utaskObj *utask.UpgradeTask) ( return options.utaskObj, err } - err = options.cleanup(openebsNamespace) - if err != nil { - return options.utaskObj, err - } - klog.Info("Upgrade Successful for", pvName) return options.utaskObj, nil } func (j *jivaVolumeOptions) cleanup(openebsNamespace string) error { + var err error + if j.replicaObj.version == currentVersion && j.replicaObj.name != "" { + klog.Info("cleaning old replica deployment") + err = deployClient.WithNamespace(j.ns).Delete(j.replicaObj.name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } + if j.controllerObj.version == currentVersion && j.ns != openebsNamespace { + klog.Info("cleaning old replica deployment") + err = deployClient.WithNamespace(j.ns).Delete(j.controllerObj.name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } return nil } -func (j *jivaVolumeOptions) preMigration(openebsNamespace string) error { - return nil +func (j *jivaVolumeOptions) migrate(pvName, openebsNamespace string) error { + err := j.migrateReplica(openebsNamespace) + if err != nil { + return err + } + err = j.migrateTarget(pvName, openebsNamespace) + return err } func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { @@ -730,10 +776,6 @@ func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { j.replicaObj.replicas[replicaDeploy.Name] = replicaDeploy.Labels["openebs.io/version"] } } - err = deployClient.WithNamespace(j.ns).Delete(j.replicaObj.name, &metav1.DeleteOptions{}) - if err != nil { - return err - } return nil } @@ -771,50 +813,44 @@ func (j *jivaVolumeOptions) migrateTarget(pvName, openebsNamespace string) error return err } } + return nil +} - // get the controller service in openebs namespace - _, err = serviceClient.WithNamespace(openebsNamespace). - Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) - if err != nil && !k8serror.IsNotFound(err) { - return err - } - // if the service is not found in openebs namespace migrate it - if k8serror.IsNotFound(err) { - svcList, err := serviceClient.WithNamespace(j.ns).List(metav1.ListOptions{ - LabelSelector: ctrlSVCLabel, - }) - if err != nil { +func (j *jivaVolumeOptions) migrateTargetSVC(pvName, openebsNamespace string) error { + // migrate service only if service not in openebs namespace + if j.ns != openebsNamespace { + // get the original service and if present remove it + svcObj, err := serviceClient.WithNamespace(j.ns). + Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) + if err != nil && !k8serror.IsNotFound(err) { return err } - svcObj := &corev1.Service{} - // if service is found in pvc namespace then recreate using it - // otherwise create the service using pv - if len(svcList.Items) != 0 { - svcObj = &svcList.Items[0] - svcObj.Namespace = openebsNamespace - svcObj.ResourceVersion = "" + if err == nil { klog.Infof("removing controller service %s in %s namespace", svcObj.Name, j.ns) err = serviceClient.WithNamespace(j.ns).Delete(svcObj.Name, &metav1.DeleteOptions{}) if err != nil { return err } - } else { - svcObj, err = getTargetSVC(pvName, openebsNamespace) + } + // get the controller service in openebs namespace + _, err = serviceClient.WithNamespace(openebsNamespace). + Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + // if the service is not found in openebs namespace create it + if k8serror.IsNotFound(err) { + svcObj, err := getTargetSVC(pvName, openebsNamespace) + if err != nil { + return err + } + klog.Infof("creating controller service %s in %s namespace", svcObj.Name, openebsNamespace) + svcObj, err = serviceClient.WithNamespace(openebsNamespace).Create(svcObj) if err != nil { return err } - svcObj.Labels["openebs.io/cas-template-name"] = deployObj.Labels["openebs.io/cas-template-name"] - } - klog.Infof("creating controller service %s in %s namespace", svcObj.Name, openebsNamespace) - svcObj, err = serviceClient.WithNamespace(openebsNamespace).Create(svcObj) - if err != nil { - return err } } - err = deployClient.WithNamespace(j.ns).Delete(j.controllerObj.name, &metav1.DeleteOptions{}) - if err != nil && !k8serror.IsNotFound(err) { - return err - } return nil } @@ -824,17 +860,12 @@ func getTargetSVC(pvName, openebsNamespace string) (*corev1.Service, error) { return nil, err } storageClass := pvObj.Spec.StorageClassName - storageClassObj, err := sc.NewKubeClient().Get(storageClass, metav1.GetOptions{}) - if err != nil { - return nil, err - } svcObj := &corev1.Service{} svcObj.ObjectMeta = metav1.ObjectMeta{ Name: pvName + "-ctrl-svc", Annotations: map[string]string{ "openebs.io/storage-class-ref": `| - name: ` + storageClass + `\n` + - `resourceVersion: ` + storageClassObj.ResourceVersion, + name: ` + storageClass, }, Labels: map[string]string{ "openebs.io/storage-engine-type": "jiva", From b8e89090e2a586e3324dfb1667243ae0d9404261 Mon Sep 17 00:00:00 2001 From: shubham Date: Mon, 6 Apr 2020 13:03:40 +0530 Subject: [PATCH 4/5] adressed review comments Signed-off-by: shubham --- pkg/upgrade/upgrader/helper.go | 48 ++-- pkg/upgrade/upgrader/jiva_upgrade.go | 371 +++++++++++++-------------- 2 files changed, 202 insertions(+), 217 deletions(-) diff --git a/pkg/upgrade/upgrader/helper.go b/pkg/upgrade/upgrader/helper.go index 3d2c909fbb..47ff3bc0a6 100644 --- a/pkg/upgrade/upgrader/helper.go +++ b/pkg/upgrade/upgrader/helper.go @@ -23,7 +23,6 @@ import ( apis "github.com/openebs/maya/pkg/apis/openebs.io/upgrade/v1alpha1" deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" - pod "github.com/openebs/maya/pkg/kubernetes/pod/v1alpha1" templates "github.com/openebs/maya/pkg/upgrade/templates/v1" utask "github.com/openebs/maya/pkg/upgrade/v1alpha2" retry "github.com/openebs/maya/pkg/util/retry" @@ -61,6 +60,23 @@ func getOpenEBSVersion(d *appsv1.Deployment) (string, error) { return d.Labels["openebs.io/version"], nil } +func checkOpenEBSVersion(d *appsv1.Deployment) (string, error) { + version, err := getOpenEBSVersion(d) + if err != nil { + return "", err + } + if (version != currentVersion) && (version != upgradeVersion) { + return "", errors.Errorf( + "replica %s version %s is neither %s nor %s\n", + d.Name, + version, + currentVersion, + upgradeVersion, + ) + } + return version, nil +} + func patchDelpoyment( deployName, namespace string, @@ -355,33 +371,3 @@ func scaleDeploy(name, namespace, label string, rc int) error { } return nil } - -func waitUntilPodsAreRunning(namespace, label string, rc int) error { - pods := &corev1.PodList{} - count := 0 - var err error - // Wait for up to 5 minutes for deployment pods to reach desired replicaCount. - for i := 0; i < 60; i++ { - // GetPodRunningCount gives number of pods running currently - pods, err = podClient. - WithNamespace(namespace). - List(metav1.ListOptions{LabelSelector: label}) - if err != nil { - return err - } - count = pod. - ListBuilderForAPIList(pods). - WithFilter(pod.IsRunning()). - List(). - Len() - if count != rc { - time.Sleep(time.Second * 5) - } else { - break - } - } - if count != rc { - return errors.Errorf("expected running pods: %d, found: %d", rc, count) - } - return nil -} diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index 785cca7f28..2e46fab602 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -118,47 +118,43 @@ func getControllerPatchDetails(d *appsv1.Deployment) ( return patchDetails, nil } +func getPatchDetailsForDeploy(pvName string, deployObj *appsv1.Deployment) (*replicaPatchDetails, error) { + patchDetails, err := getReplicaPatchDetails(deployObj) + if err != nil { + return nil, err + } + patchDetails.UpgradeVersion = upgradeVersion + patchDetails.PVName = pvName + return patchDetails, nil +} + func getReplica(pvName, replicaLabel, namespace, openebsNamespace string) (*replicaDetails, error) { replicaObj := &replicaDetails{ replicas: map[string]string{}, } + var err error // check if old replica is present for currentVersion < 1.9.0 // if present then migration is not complete and store the old // replica details if currentVersion < "1.9.0" { deployObj, err := deployClient.WithNamespace(namespace).Get(pvName + "-rep") + if err != nil && !k8serror.IsNotFound(err) { return nil, errors.Wrapf(err, "failed to get replica deployment") } if err == nil { - klog.Info("getting old deployment") - if deployObj.Name == "" { - return nil, errors.Errorf("missing deployment name for replica") - } - replicaObj.name = deployObj.Name - version, err := getOpenEBSVersion(deployObj) + version, err := checkOpenEBSVersion(deployObj) if err != nil { return nil, err } - if (version != currentVersion) && (version != upgradeVersion) { - return nil, errors.Errorf( - "replica version %s is neither %s nor %s\n", - version, - currentVersion, - upgradeVersion, - ) - } - replicaObj.version = version - patchDetails, err := getReplicaPatchDetails(deployObj) + replicaObj.patchDetails, err = getPatchDetailsForDeploy(pvName, deployObj) if err != nil { return nil, err } - replicaObj.patchDetails = patchDetails - replicaObj.patchDetails.UpgradeVersion = upgradeVersion - replicaObj.patchDetails.PVName = pvName + replicaObj.name = deployObj.Name + replicaObj.version = version } } - // replicaList, err := deployClient.WithNamespace(openebsNamespace).List(&metav1.ListOptions{ LabelSelector: replicaLabel, }) @@ -170,27 +166,17 @@ func getReplica(pvName, replicaLabel, namespace, openebsNamespace string) (*repl // be removed and not patched if replica.Name != pvName+"-rep" { deployObj := &replica - version, err := getOpenEBSVersion(deployObj) + version, err := checkOpenEBSVersion(deployObj) if err != nil { return nil, err } - if (version != currentVersion) && (version != upgradeVersion) { - return nil, errors.Errorf( - "replica %s version %s is neither %s nor %s\n", - deployObj.Name, - version, - currentVersion, - upgradeVersion, - ) - } replicaObj.replicas[deployObj.Name] = version - patchDetails, err := getReplicaPatchDetails(deployObj) - if err != nil { - return nil, err + if replicaObj.patchDetails == nil { + replicaObj.patchDetails, err = getPatchDetailsForDeploy(pvName, deployObj) + if err != nil { + return nil, err + } } - replicaObj.patchDetails = patchDetails - replicaObj.patchDetails.UpgradeVersion = upgradeVersion - replicaObj.patchDetails.PVName = pvName } } return replicaObj, nil @@ -229,31 +215,31 @@ func getController(controllerLabel, namespace string) (*controllerDetails, error } func patchReplica(name, namespace string, replicaObj *replicaDetails) error { - if replicaObj.replicas[name] == currentVersion { - tmpl, err := template.New("replicaPatch"). - Parse(templates.JivaReplicaPatch) - if err != nil { - return errors.Wrapf(err, "failed to create template for replica patch") - } - err = tmpl.Execute(&buffer, replicaObj.patchDetails) - if err != nil { - return errors.Wrapf(err, "failed to populate template for replica patch") - } - replicaPatch := buffer.String() - buffer.Reset() - err = patchDelpoyment( - name, - namespace, - types.StrategicMergePatchType, - []byte(replicaPatch), - ) - if err != nil { - return errors.Wrapf(err, "failed to patch replica deployment %s", name) - } - klog.Infof("%s patched", name) - } else { + if replicaObj.replicas[name] == upgradeVersion { klog.Infof("replica deployment %s already in %s version", name, upgradeVersion) + return nil + } + tmpl, err := template.New("replicaPatch"). + Parse(templates.JivaReplicaPatch) + if err != nil { + return errors.Wrapf(err, "failed to create template for replica patch") } + err = tmpl.Execute(&buffer, replicaObj.patchDetails) + if err != nil { + return errors.Wrapf(err, "failed to populate template for replica patch") + } + replicaPatch := buffer.String() + buffer.Reset() + err = patchDelpoyment( + name, + namespace, + types.StrategicMergePatchType, + []byte(replicaPatch), + ) + if err != nil { + return errors.Wrapf(err, "failed to patch replica deployment %s", name) + } + klog.Infof("%s patched", name) return nil } @@ -479,6 +465,21 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } +func (j *jivaVolumeOptions) preReplicaUpgradeLessThan190(pvName, openebsNamespace string) (string, error) { + if currentVersion < "1.9.0" { + err := scaleDeploy(j.replicaObj.name, j.ns, replicaDeployLabel, 0) + if err != nil { + return "failed to get scale down replica deployment", err + + } + err = j.migrateTargetSVC(pvName, openebsNamespace) + if err != nil { + return "failed to get migrate target service", err + } + } + return "", nil +} + func (j *jivaVolumeOptions) replicaUpgrade(pvName, openebsNamespace string) error { var err, uerr error statusObj := utask.UpgradeDetailedStatuses{Step: utask.ReplicaUpgrade} @@ -487,40 +488,28 @@ func (j *jivaVolumeOptions) replicaUpgrade(pvName, openebsNamespace string) erro if uerr != nil && isENVPresent { return uerr } - statusObj.Phase = utask.StepErrored - if currentVersion < "1.9.0" { - err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) - if err != nil { - statusObj.Message = "failed to get scale down target deployment" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return err - } - err = scaleDeploy(j.replicaObj.name, j.ns, replicaDeployLabel, 0) - if err != nil { - statusObj.Message = "failed to get scale down replica deployment" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return err + err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) + if err != nil { + statusObj.Message = "failed to get scale down target deployment" + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr } - err = j.migrateTargetSVC(pvName, openebsNamespace) - if err != nil { - statusObj.Message = "failed to get migrate target service" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return err + return err + } + + msg, err := j.preReplicaUpgradeLessThan190(pvName, openebsNamespace) + if err != nil { + statusObj.Message = msg + statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) + j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) + if uerr != nil && isENVPresent { + return uerr } + return err } for name := range j.replicaObj.replicas { @@ -559,18 +548,6 @@ func (j *jivaVolumeOptions) targetUpgrade(pvName, openebsNamespace string) error statusObj.Phase = utask.StepErrored - if currentVersion < "1.9.0" && j.ns != openebsNamespace { - err = j.migrateTarget(pvName, openebsNamespace) - if err != nil { - statusObj.Message = "failed to migrate target to openebs namespace" - statusObj.Reason = strings.Replace(err.Error(), ":", "", -1) - j.utaskObj, uerr = updateUpgradeDetailedStatus(j.utaskObj, statusObj, openebsNamespace) - if uerr != nil && isENVPresent { - return uerr - } - return err - } - } // controller patch err = patchController(j.controllerObj, openebsNamespace) if err != nil { @@ -727,15 +704,45 @@ func (j *jivaVolumeOptions) cleanup(openebsNamespace string) error { } func (j *jivaVolumeOptions) migrate(pvName, openebsNamespace string) error { - err := j.migrateReplica(openebsNamespace) - if err != nil { - return err + var err error + // if old replica is missing then migration was + // successful till replica cleanup in previous iteration + if j.replicaObj.name != "" { + err = j.migrateReplica(openebsNamespace) + if err != nil { + return err + } + } + // if pvc deployed in openebs namespace no need + // to migrate the controller deployment + if j.ns != openebsNamespace { + err = j.migrateTarget(pvName, openebsNamespace) } - err = j.migrateTarget(pvName, openebsNamespace) return err } +func getNodeNames(deployObj *appsv1.Deployment) (int, []string) { + matchExp := deployObj.Spec.Template.Spec.Affinity.NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions + for i, exp := range matchExp { + if exp.Key == "kubernetes.io/hostname" { + return i, exp.Values + } + } + return -1, nil +} + func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { + // get the old replica deployment by name + oldDeployObj, err := deployClient.WithNamespace(j.ns).Get(j.replicaObj.name) + if err != nil { + return err + } + index, nodeNames := getNodeNames(oldDeployObj) + if index == -1 { + return errors.New("unable to find kubernetes.io/hostname key in nodeAffinity") + } + replicaCount := len(nodeNames) // get the separate replica deployments in openebs namespace deployList, err := deployClient.WithNamespace(openebsNamespace).List(&metav1.ListOptions{ LabelSelector: replicaDeployLabel, @@ -743,112 +750,104 @@ func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { if err != nil { return err } - // get the old replica deployment by name - oldDeployObj, err := deployClient.WithNamespace(j.ns).Get(j.replicaObj.name) - if err != nil { - return err - } - replicaCount := int(*oldDeployObj.Spec.Replicas) - replicasLeft := len(deployList.Items) + replicasCreated := len(deployList.Items) // if the volume was deployed in openebs namespace while provisioning // as the old deployment also has the same label if j.ns == openebsNamespace { - replicasLeft = replicasLeft - 1 - } - if len(deployList.Items) != replicaCount { - klog.Infof("splitting replica deployment") - nodeNames := oldDeployObj.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values - var zero int32 - for i := replicasLeft; i < replicaCount; i++ { - replicaDeploy := oldDeployObj.DeepCopy() - replicaDeploy.Name = replicaDeploy.Name + "-" + strconv.Itoa(i+1) - replicaDeploy.Namespace = openebsNamespace - replicaDeploy.ResourceVersion = "" - replicaDeploy.Spec.Replicas = &zero - replicaDeploy.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values = []string{nodeNames[i]} - klog.Infof("creating replica deployment %s in %s namespace", replicaDeploy.Name, openebsNamespace) - replicaDeploy, err := deployClient.WithNamespace(openebsNamespace).Create(replicaDeploy) - if err != nil { - return err - } - j.replicaObj.replicas[replicaDeploy.Name] = replicaDeploy.Labels["openebs.io/version"] + replicasCreated = replicasCreated - 1 + } + + klog.Infof("splitting replica deployment") + var zero int32 + for i := replicasCreated; i < replicaCount; i++ { + replicaDeploy := oldDeployObj.DeepCopy() + replicaDeploy.Name = replicaDeploy.Name + "-" + strconv.Itoa(i+1) + replicaDeploy.Namespace = openebsNamespace + replicaDeploy.ResourceVersion = "" + replicaDeploy.Spec.Replicas = &zero + replicaDeploy.Spec.Template.Spec.Affinity.NodeAffinity. + RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[index].Values = []string{nodeNames[i]} + klog.Infof("creating replica deployment %s in %s namespace", replicaDeploy.Name, openebsNamespace) + replicaDeploy, err := deployClient.WithNamespace(openebsNamespace).Create(replicaDeploy) + if err != nil { + return err } + j.replicaObj.replicas[replicaDeploy.Name] = replicaDeploy.Labels["openebs.io/version"] } + return nil } func (j *jivaVolumeOptions) migrateTarget(pvName, openebsNamespace string) error { // get the controller deployment in openebs namespace + // controllerObj.name cannot be nil as after successful upgrade + // controller is removed and no deploy or svc is present in pvc namespace + // so controllerObj will be in openebs namespace deployObj, err := deployClient.WithNamespace(openebsNamespace).Get(j.controllerObj.name) + if err == nil { + klog.Info("controller deployment already migrated to openebs namespace") + return nil + } if err != nil && !k8serror.IsNotFound(err) { return err } // if the deployment is not found in openebs namespace migrate it - if k8serror.IsNotFound(err) { - var zero int32 - deployObj, err = deployClient.WithNamespace(j.ns).Get(j.controllerObj.name) - if err != nil { - return err - } - deployObj.Namespace = openebsNamespace - deployObj.ResourceVersion = "" - deployObj.Spec.Replicas = &zero - // if target-affinity is set for the pvc them openebs namespace - // needs to be added as a bug fix. - if deployObj.Spec.Template.Spec.Affinity != nil { - if deployObj.Spec.Template.Spec.Affinity.PodAffinity. - RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector. - MatchExpressions[0].Key == "openebs.io/target-affinity" { - deployObj.Spec.Template.Spec.Affinity.PodAffinity. - RequiredDuringSchedulingIgnoredDuringExecution[0]. - Namespaces = []string{j.ns} - } - } - klog.Infof("creating controller deployment %s in %s namespace", deployObj.Name, openebsNamespace) - deployObj, err = deployClient.WithNamespace(openebsNamespace).Create(deployObj) - if err != nil { - return err - } + var zero int32 + deployObj, err = deployClient.WithNamespace(j.ns).Get(j.controllerObj.name) + if err != nil { + return err } - return nil + deployObj.Namespace = openebsNamespace + deployObj.ResourceVersion = "" + deployObj.Spec.Replicas = &zero + // if target-affinity is set for the pvc them openebs namespace + // needs to be added as a bug fix. + if deployObj.Spec.Template.Spec.Affinity != nil { + deployObj.Spec.Template.Spec.Affinity.PodAffinity. + RequiredDuringSchedulingIgnoredDuringExecution[0]. + Namespaces = []string{j.ns} + } + + klog.Infof("creating controller deployment %s in %s namespace", deployObj.Name, openebsNamespace) + _, err = deployClient.WithNamespace(openebsNamespace).Create(deployObj) + return err } func (j *jivaVolumeOptions) migrateTargetSVC(pvName, openebsNamespace string) error { // migrate service only if service not in openebs namespace - if j.ns != openebsNamespace { - // get the original service and if present remove it - svcObj, err := serviceClient.WithNamespace(j.ns). - Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) - if err != nil && !k8serror.IsNotFound(err) { + if j.ns == openebsNamespace { + return nil + } + // get the original service and if present remove it + svcObj, err := serviceClient.WithNamespace(j.ns). + Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + if err == nil { + klog.Infof("removing controller service %s in %s namespace", svcObj.Name, j.ns) + err = serviceClient.WithNamespace(j.ns).Delete(svcObj.Name, &metav1.DeleteOptions{}) + if err != nil { return err } - if err == nil { - klog.Infof("removing controller service %s in %s namespace", svcObj.Name, j.ns) - err = serviceClient.WithNamespace(j.ns).Delete(svcObj.Name, &metav1.DeleteOptions{}) - if err != nil { - return err - } - } - // get the controller service in openebs namespace - _, err = serviceClient.WithNamespace(openebsNamespace). - Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) - if err != nil && !k8serror.IsNotFound(err) { + } + // get the controller service in openebs namespace + _, err = serviceClient.WithNamespace(openebsNamespace). + Get(j.controllerObj.name+"-svc", metav1.GetOptions{}) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + // if the service is not found in openebs namespace create it + if k8serror.IsNotFound(err) { + svcObj, err := getTargetSVC(pvName, openebsNamespace) + if err != nil { return err } - // if the service is not found in openebs namespace create it - if k8serror.IsNotFound(err) { - svcObj, err := getTargetSVC(pvName, openebsNamespace) - if err != nil { - return err - } - klog.Infof("creating controller service %s in %s namespace", svcObj.Name, openebsNamespace) - svcObj, err = serviceClient.WithNamespace(openebsNamespace).Create(svcObj) - if err != nil { - return err - } + klog.Infof("creating controller service %s in %s namespace", svcObj.Name, openebsNamespace) + svcObj, err = serviceClient.WithNamespace(openebsNamespace).Create(svcObj) + if err != nil { + return err } } return nil From 607c73261121baecab860320f6a5675186dac520 Mon Sep 17 00:00:00 2001 From: shubham Date: Mon, 6 Apr 2020 21:26:53 +0530 Subject: [PATCH 5/5] update function comments and names of variables & functions Signed-off-by: shubham --- pkg/upgrade/upgrader/helper.go | 17 --------- pkg/upgrade/upgrader/jiva_upgrade.go | 52 ++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/pkg/upgrade/upgrader/helper.go b/pkg/upgrade/upgrader/helper.go index 47ff3bc0a6..ccad9df076 100644 --- a/pkg/upgrade/upgrader/helper.go +++ b/pkg/upgrade/upgrader/helper.go @@ -60,23 +60,6 @@ func getOpenEBSVersion(d *appsv1.Deployment) (string, error) { return d.Labels["openebs.io/version"], nil } -func checkOpenEBSVersion(d *appsv1.Deployment) (string, error) { - version, err := getOpenEBSVersion(d) - if err != nil { - return "", err - } - if (version != currentVersion) && (version != upgradeVersion) { - return "", errors.Errorf( - "replica %s version %s is neither %s nor %s\n", - d.Name, - version, - currentVersion, - upgradeVersion, - ) - } - return version, nil -} - func patchDelpoyment( deployName, namespace string, diff --git a/pkg/upgrade/upgrader/jiva_upgrade.go b/pkg/upgrade/upgrader/jiva_upgrade.go index 2e46fab602..bf8c02c676 100644 --- a/pkg/upgrade/upgrader/jiva_upgrade.go +++ b/pkg/upgrade/upgrader/jiva_upgrade.go @@ -118,7 +118,7 @@ func getControllerPatchDetails(d *appsv1.Deployment) ( return patchDetails, nil } -func getPatchDetailsForDeploy(pvName string, deployObj *appsv1.Deployment) (*replicaPatchDetails, error) { +func getPatchDetailsForReplicaDeploy(pvName string, deployObj *appsv1.Deployment) (*replicaPatchDetails, error) { patchDetails, err := getReplicaPatchDetails(deployObj) if err != nil { return nil, err @@ -128,7 +128,24 @@ func getPatchDetailsForDeploy(pvName string, deployObj *appsv1.Deployment) (*rep return patchDetails, nil } -func getReplica(pvName, replicaLabel, namespace, openebsNamespace string) (*replicaDetails, error) { +func validateReplicaDeployVersion(d *appsv1.Deployment) (string, error) { + version, err := getOpenEBSVersion(d) + if err != nil { + return "", err + } + if (version != currentVersion) && (version != upgradeVersion) { + return "", errors.Errorf( + "replica %s version %s is neither %s nor %s\n", + d.Name, + version, + currentVersion, + upgradeVersion, + ) + } + return version, nil +} + +func getReplica(pvName, replicaLabel, volumeNamespace, openebsNamespace string) (*replicaDetails, error) { replicaObj := &replicaDetails{ replicas: map[string]string{}, } @@ -136,18 +153,21 @@ func getReplica(pvName, replicaLabel, namespace, openebsNamespace string) (*repl // check if old replica is present for currentVersion < 1.9.0 // if present then migration is not complete and store the old // replica details + // replicaObj.name and replicaObj.version would be empty if old replica got + // deleted as part of upgrade. + // So, later on code uses replicaObj.name to perform replica related migration. if currentVersion < "1.9.0" { - deployObj, err := deployClient.WithNamespace(namespace).Get(pvName + "-rep") + deployObj, err := deployClient.WithNamespace(volumeNamespace).Get(pvName + "-rep") if err != nil && !k8serror.IsNotFound(err) { return nil, errors.Wrapf(err, "failed to get replica deployment") } if err == nil { - version, err := checkOpenEBSVersion(deployObj) + version, err := validateReplicaDeployVersion(deployObj) if err != nil { return nil, err } - replicaObj.patchDetails, err = getPatchDetailsForDeploy(pvName, deployObj) + replicaObj.patchDetails, err = getPatchDetailsForReplicaDeploy(pvName, deployObj) if err != nil { return nil, err } @@ -166,13 +186,14 @@ func getReplica(pvName, replicaLabel, namespace, openebsNamespace string) (*repl // be removed and not patched if replica.Name != pvName+"-rep" { deployObj := &replica - version, err := checkOpenEBSVersion(deployObj) + version, err := validateReplicaDeployVersion(deployObj) if err != nil { return nil, err } + // replicaObj.replicas[deployObj.Name] = version if replicaObj.patchDetails == nil { - replicaObj.patchDetails, err = getPatchDetailsForDeploy(pvName, deployObj) + replicaObj.patchDetails, err = getPatchDetailsForReplicaDeploy(pvName, deployObj) if err != nil { return nil, err } @@ -465,8 +486,14 @@ func (j *jivaVolumeOptions) preupgrade(pvName, openebsNamespace string) error { return nil } +// preReplicaUpgradeLessThan190 scales down old replica deployment +// and migrates the target service to openebs namespace +// before bringing up the new separate deployments func (j *jivaVolumeOptions) preReplicaUpgradeLessThan190(pvName, openebsNamespace string) (string, error) { - if currentVersion < "1.9.0" { + // if the upgrade is successful till replica cleanup and restarts + // after that old replica will be missing and if replica cleanup + // was done then service was also migrated successfully + if currentVersion < "1.9.0" && j.replicaObj.name != "" { err := scaleDeploy(j.replicaObj.name, j.ns, replicaDeployLabel, 0) if err != nil { return "failed to get scale down replica deployment", err @@ -490,6 +517,8 @@ func (j *jivaVolumeOptions) replicaUpgrade(pvName, openebsNamespace string) erro } statusObj.Phase = utask.StepErrored + // Scaling down controller ensures no I/O occurs + // which make volume to come in RW mode early err = scaleDeploy(j.controllerObj.name, j.ns, ctrlDeployLabel, 0) if err != nil { statusObj.Message = "failed to get scale down target deployment" @@ -694,7 +723,7 @@ func (j *jivaVolumeOptions) cleanup(openebsNamespace string) error { } } if j.controllerObj.version == currentVersion && j.ns != openebsNamespace { - klog.Info("cleaning old replica deployment") + klog.Info("cleaning old controller deployment") err = deployClient.WithNamespace(j.ns).Delete(j.controllerObj.name, &metav1.DeleteOptions{}) if err != nil { return err @@ -757,6 +786,9 @@ func (j *jivaVolumeOptions) migrateReplica(openebsNamespace string) error { replicasCreated = replicasCreated - 1 } + // replica deployment pv-name-rep will be split into multiple replicas like + // pv-name-rep-1, pv-name-rep-2,... pv-name-rep-n, + // where n is the replica count for this volume. klog.Infof("splitting replica deployment") var zero int32 for i := replicasCreated; i < replicaCount; i++ { @@ -864,7 +896,7 @@ func getTargetSVC(pvName, openebsNamespace string) (*corev1.Service, error) { Name: pvName + "-ctrl-svc", Annotations: map[string]string{ "openebs.io/storage-class-ref": `| - name: ` + storageClass, + name: ` + storageClass, }, Labels: map[string]string{ "openebs.io/storage-engine-type": "jiva",