Skip to content

Commit

Permalink
add extra check to verify node to test is still schedulable
Browse files Browse the repository at this point in the history
  • Loading branch information
maciekm committed Apr 25, 2024
1 parent a9e0a57 commit e870791
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 14 deletions.
41 changes: 33 additions & 8 deletions cmd/storage-check/run_check.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func runStorageCheck() {
// Create a storage resource.
storageConfig := createStorageConfig(checkStorageName)
log.Infoln("Created Storage resource.")
log.Infof("It looks like: %v", storageConfig)
log.Debugf("It looks like: %v", storageConfig)

// Apply the storage struct manifest to the cluster.
var storageResult StorageResult
Expand Down Expand Up @@ -100,7 +100,7 @@ func runStorageCheck() {
//initStorageConfig := initializeStorageConfig(checkStorageName+"-init", checkStorageName)
initStorageConfig := initializeStorageConfig(checkStorageName+"-init-job", checkStorageName)
log.Infoln("Created Storage Initialiazer resource.")
log.Infof("It looks like: %v", initStorageConfig)
log.Debugf("It looks like: %v", initStorageConfig)

// Initialize the storage
var initStorageResult InitStorageResult
Expand Down Expand Up @@ -157,13 +157,13 @@ func runStorageCheck() {
// LabelSelector: defaultLabelKey + "=" + defaultLabelValueBase + strconv.Itoa(int(now.Unix())),
})
if err != nil {
log.Infoln("Error on getting nodes..not sure what to do", err)
log.Errorln("Error on getting nodes.. not sure what to do", err)
}

log.Infoln("Nodes are ", nodes)
log.Debugln("Nodes are ", nodes)
for _, n := range nodes.Items {
log.Infoln("Node.name=", n.Name)
log.Infoln("Status=", n.Status)
log.Debugln("Status=", n.Status)

node := new(Node)
node.name = n.Name
Expand All @@ -179,7 +179,7 @@ func runStorageCheck() {
}
log.Printf("Adding node %s with taints %s to %s scheduled for check", n.Name, formatTaints(n.Spec.Taints), status)
} else {
log.Infoln("Adding untainted node ", n.Name, " to be scheduled for check")
log.Infoln("Adding untainted node", n.Name, "to be scheduled for check")
node.schedulable = true
}
checkNodes[node.name] = node
Expand Down Expand Up @@ -221,10 +221,15 @@ func runStorageCheck() {

for nodeName, node := range checkNodes {
if node.schedulable == true {
log.Infof("Creating config for %s %+v", nodeName, node)
if !verifyStillSchedulable(nodeName) {
log.Infof("Node %s is no longer schedulable, skipping check", nodeName)
continue
}

log.Infof("Creating config for %s", nodeName)
config := checkNodeConfig(checkStorageName+"-check-job", checkStorageName, nodeName)
log.Infoln("Created config.")
log.Infof("It looks like: %+v", config)
log.Debugf("It looks like: %+v", config)

// Initialize the storage
var checkStorageResult CheckStorageResult
Expand Down Expand Up @@ -401,3 +406,23 @@ func formatTaints(taints []v1.Taint) string {

return strings.Join(taintStrings, ",")
}

func verifyStillSchedulable(nodeName string) bool {
log.Infof("Checking if %s is still schedulable", nodeName)

nodes, err := client.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
log.Errorln("Error on getting nodes... not sure what to do", err)
}

schedulable := false

for _, n := range nodes.Items {
if n.Name == nodeName {
schedulable = true
break
}
}

return schedulable
}
12 changes: 6 additions & 6 deletions cmd/storage-check/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func createStorageConfig(pvcname string) *corev1.PersistentVolumeClaim {
// Add the storage spec to the storage.
pvc.Spec = pvcSpec

log.Infoln("PVC ", pvcname, " is", pvc, "namespace environment variables:chris", additionalEnvVars)
log.Debugln("PVC", pvcname, "is", pvc, "namespace environment variables:chris", additionalEnvVars)
return pvc
}

Expand Down Expand Up @@ -154,7 +154,7 @@ func initializeStorageConfig(jobName string, pvcName string) *batchv1.Job {
// Add the storage spec to the storage.
job.Spec = jobSpec

log.Infoln("Job ", jobName, " is", job, "namespace environment variables:", additionalEnvVars)
log.Debugln("Job", jobName, "is", job, "namespace environment variables:", additionalEnvVars)
return job
}

Expand All @@ -164,7 +164,7 @@ func checkNodeConfig(jobName string, pvcName string, node string) *batchv1.Job {
// Make a Job
job := &batchv1.Job{}

log.Infoln("Creating a job", jobName, " in", checkNamespace, "namespace environment variables:", additionalEnvVars)
log.Infoln("Creating a job", jobName, "in", checkNamespace, "namespace environment variables:", additionalEnvVars)

// Make labels for Job
labels := make(map[string]string, 0)
Expand Down Expand Up @@ -214,7 +214,7 @@ func checkNodeConfig(jobName string, pvcName string, node string) *batchv1.Job {
// Add the storage spec to the storage.
job.Spec = jobSpec

log.Infoln("Job ", jobName, " is", job, "namespace environment variables:", additionalEnvVars)
log.Debugln("Job", jobName, "is", job, "namespace environment variables:", additionalEnvVars)
return job
}

Expand Down Expand Up @@ -623,7 +623,7 @@ func deleteStorageCheckAndWait(ctx context.Context) error {
// Send a delete on the storage.
err := deleteStorageCheckJob(jobName)
if err != nil {
log.Infoln("Could not delete storage init job :", checkStorageName)
log.Infoln("Could not delete storage init job:", checkStorageName)
}

return <-deleteChan
Expand Down Expand Up @@ -689,7 +689,7 @@ func deleteStorageInitJobAndWait(ctx context.Context) error {
// Send a delete on the storage.
err := deleteStorageInitJob(jobName)
if err != nil {
log.Infoln("Could not delete storage init job :", checkStorageName)
log.Infoln("Could not delete storage init job:", checkStorageName)
}

return <-deleteChan
Expand Down

0 comments on commit e870791

Please sign in to comment.