Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Csm object fails during down scale #245

Merged
merged 1 commit into from
May 19, 2023
Merged

Csm object fails during down scale #245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion controllers/csm_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync/atomic"
"time"

Expand Down Expand Up @@ -357,18 +358,25 @@ func (r *ContainerStorageModuleReconciler) handleDeploymentUpdate(oldObj interfa

csm := new(csmv1.ContainerStorageModule)
err := r.Client.Get(ctx, namespacedName, csm)

if err != nil {
log.Error("deployment get csm", "error", err.Error())
}

newStatus := csm.GetCSMStatus()

// Updating controller status manually as controller runtime API is not updating csm object with latest data
// TODO: Can remove this once the controller runtime repo has a fix for updating the object passed
newStatus.ControllerStatus.Available = strconv.Itoa(int(available))
newStatus.ControllerStatus.Desired = strconv.Itoa(int(desired))
newStatus.ControllerStatus.Failed = strconv.Itoa(int(numberUnavailable))

err = utils.UpdateStatus(ctx, csm, r, newStatus)
if err != nil {
log.Debugw("deployment status ", "pods", err.Error())
} else {
r.EventRecorder.Eventf(csm, corev1.EventTypeNormal, csmv1.EventCompleted, "Driver deployment running OK")
}

}

func (r *ContainerStorageModuleReconciler) handlePodsUpdate(oldObj interface{}, obj interface{}) {
Expand Down
43 changes: 32 additions & 11 deletions pkg/utils/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ func getDeploymentStatus(ctx context.Context, instance *csmv1.ContainerStorageMo
err = errors.New(msg)
}

log.Infof("Deployment totalReplicas count %d totalReadyPods %d totalFailedCount %d", totalReplicas, totalReadyPods, totalFailedCount)

return totalReplicas, csmv1.PodStatus{
Available: fmt.Sprintf("%d", totalReadyPods),
Desired: fmt.Sprintf("%d", totalReplicas),
Expand Down Expand Up @@ -221,33 +223,46 @@ func getDaemonSetStatus(ctx context.Context, instance *csmv1.ContainerStorageMod
func calculateState(ctx context.Context, instance *csmv1.ContainerStorageModule, r ReconcileCSM, newStatus *csmv1.ContainerStorageModuleStatus) (bool, error) {
log := logger.GetLogger(ctx)
running := false
controllerReplicas, controllerStatus, controllerErr := getDeploymentStatus(ctx, instance, r)
// TODO: Currently commented this block of code as the API used to get the latest deployment status is not working as expected
// TODO: Can be uncommented once this issues gets sorted out
/* controllerReplicas, controllerStatus, controllerErr := getDeploymentStatus(ctx, instance, r)
expected, nodeStatus, daemonSetErr := getDaemonSetStatus(ctx, instance, r)
newStatus.ControllerStatus = controllerStatus
newStatus.NodeStatus = nodeStatus */
expected, nodeStatus, daemonSetErr := getDaemonSetStatus(ctx, instance, r)
newStatus.NodeStatus = nodeStatus
controllerReplicas := newStatus.ControllerStatus.Desired
controllerStatus := newStatus.ControllerStatus

newStatus.State = constants.Failed
log.Infof("deployment controllerReplicas [%d]", controllerReplicas)
log.Infof("deployment controllerReplicas [%s]", controllerReplicas)
log.Infof("deployment controllerStatus.Available [%s]", controllerStatus.Available)

log.Infof("daemonset expected [%d]", expected)
log.Infof("daemonset nodeStatus.Available [%s]", nodeStatus.Available)

if (fmt.Sprintf("%d", controllerReplicas) == controllerStatus.Available) && (fmt.Sprintf("%d", expected) == nodeStatus.Available) {
if (controllerReplicas == controllerStatus.Available) && (fmt.Sprintf("%d", expected) == nodeStatus.Available) {
running = true
newStatus.State = constants.Succeeded
}
log.Infof("calculate overall state [%s]", newStatus.State)
var err error
if controllerErr != nil {
err = controllerErr
}
var err error = nil
// TODO: Uncomment this when the controller runtime API gets fixed
/*
if controllerErr != nil {
err = controllerErr
}
if daemonSetErr != nil {
err = daemonSetErr
}
if daemonSetErr != nil && controllerErr != nil {
err = fmt.Errorf("ControllerError: %s, Daemonseterror: %s", controllerErr.Error(), daemonSetErr.Error())
log.Infof("calculate overall error msg [%s]", err.Error())
} */

if daemonSetErr != nil {
err = daemonSetErr
}
if daemonSetErr != nil && controllerErr != nil {
err = fmt.Errorf("ControllerError: %s, Daemonseterror: %s", controllerErr.Error(), daemonSetErr.Error())
log.Infof("calculate overall error msg [%s]", err.Error())
log.Infof("calculate Daemonseterror msg [%s]", daemonSetErr.Error())
}
SetStatus(ctx, r, instance, newStatus)
return running, err
Expand Down Expand Up @@ -284,6 +299,12 @@ func UpdateStatus(ctx context.Context, instance *csmv1.ContainerStorageModule, r
if err != nil {
return err
}

log.Infow("instance - new controller Status", "desired", instance.Status.ControllerStatus.Desired)
log.Infow("instance - new controller Status", "Available", instance.Status.ControllerStatus.Available)
log.Infow("instance - new controller Status", "numberUnavailable", instance.Status.ControllerStatus.Failed)
log.Infow("instance - new controller Status", "State", instance.Status.State)

csm.Status = instance.Status
err = r.GetClient().Status().Update(ctx, csm)
if err != nil {
Expand Down