handle storage state out of sync with storage migration

kubernetes-sigs · Feb 18, 2020 · 25ca7ee · 25ca7ee
1 parent e1765ef
commit 25ca7ee
Showing 1 changed file with 30 additions and 5 deletions.
diff --git a/pkg/trigger/discovery_handler.go b/pkg/trigger/discovery_handler.go
@@ -185,10 +185,35 @@ func (mt *MigrationTrigger) processDiscoveryResource(r metav1.APIResource) {
 		utilruntime.HandleError(getErr)
 		return
 	}
-
-	stale := (getErr == nil && mt.staleStorageState(ss))
-	storageVersionChanged := (getErr == nil && ss.Status.CurrentStorageVersionHash != r.StorageVersionHash)
-	notFound := (getErr != nil && errors.IsNotFound(getErr))
+	stale := getErr == nil && mt.staleStorageState(ss)
+	storageVersionChanged := getErr == nil && ss.Status.CurrentStorageVersionHash != r.StorageVersionHash
+	notFound := getErr != nil && errors.IsNotFound(getErr)
+
+	var relaunchMigration bool
+	if stale || notFound || storageVersionChanged {
+		relaunchMigration = true
+	} else {
+		// get the corresponding StorageVersionMigration resource
+		migrations, err := mt.migrationInformer.GetIndexer().ByIndex(controller.ResourceIndex, controller.ToIndex(toGroupResource(r)))
+		if err != nil {
+			utilruntime.HandleError(getErr)
+			return
+		}
+		switch {
+		case len(migrations) == 0:
+			// corresponding StorageVersionMigration resource missing
+			relaunchMigration = true
+		case len(ss.Status.PersistedStorageVersionHashes) == 0:
+			// this should not happen
+			relaunchMigration = true
+		case ss.Status.PersistedStorageVersionHashes[0] == migrationv1alpha1.Unknown:
+			migration := migrations[0].(*migrationv1alpha1.StorageVersionMigration)
+			if controller.HasCondition(migration, migrationv1alpha1.MigrationFailed) {
+				// StorageVersionMigration failed, but StorageVersion never got its first update
+				relaunchMigration = true
+			}
+		}
+	}
 
 	if stale {
 		if err := mt.client.MigrationV1alpha1().StorageStates().Delete(storageStateName(toGroupResource(r)), nil); err != nil {
@@ -197,7 +222,7 @@ func (mt *MigrationTrigger) processDiscoveryResource(r metav1.APIResource) {
 		}
 	}
 
-	if stale || storageVersionChanged || notFound {
+	if relaunchMigration {
 		// Note that this means historical migration objects are deleted.
 		if err := mt.relaunchMigration(r); err != nil {
 			utilruntime.HandleError(err)