fix(rollbacks): support for tolerating some instance failures (#4144)

spinnaker · Oct 3, 2017 · 539826b · 539826b
1 parent 25b98bf
commit 539826b
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 2 deletions.
diff --git a/...scripts/modules/amazon/src/serverGroup/details/rollback/rollbackServerGroup.controller.js b/...scripts/modules/amazon/src/serverGroup/details/rollback/rollbackServerGroup.controller.js
@@ -16,13 +16,29 @@ module.exports = angular.module('spinnaker.amazon.serverGroup.details.rollback.c
       $scope.allServerGroups = allServerGroups.sort((a, b) => b.name.localeCompare(a.name));
       $scope.verification = {};
 
+      var desired = serverGroup.capacity.desired;
+
+      if (desired < 10) {
+        var healthyPercent = 100;
+      } else if (desired < 20) {
+        // accept 1 instance in an unknown state during rollback
+        healthyPercent = 90;
+      } else {
+        healthyPercent = 95;
+      }
+
       $scope.command = {
         rollbackType: 'EXPLICIT',
         rollbackContext: {
-          rollbackServerGroupName: serverGroup.name
+          rollbackServerGroupName: serverGroup.name,
+          targetHealthyRollbackPercentage: healthyPercent
         }
       };
 
+      $scope.minHealthy = function(percent) {
+        return Math.ceil(desired * percent / 100);
+      };
+
       if (application && application.attributes) {
         if (application.attributes.platformHealthOnlyShowOverride && application.attributes.platformHealthOnly) {
           $scope.command.interestingHealthProviderNames = ['Amazon'];

diff --git a/app/scripts/modules/amazon/src/serverGroup/details/rollback/rollbackServerGroup.html b/app/scripts/modules/amazon/src/serverGroup/details/rollback/rollbackServerGroup.html
@@ -30,7 +30,22 @@ <h3>Rollback {{serverGroup.name}}</h3>
         </div>
       </div>
 
-      <task-reason command="command"></task-reason>
+      <div class="row">
+        <task-reason command="command"></task-reason>
+      </div>
+
+      <div class="row">
+        <div class="col-sm-11 col-sm-offset-1">
+          Consider rollback successful when
+          <input type="number"
+                 min="0"
+                 max="100"
+                 ng-model="command.rollbackContext.targetHealthyRollbackPercentage"
+                 class="form-control input-sm inline-number"
+          />
+          percent of instances are healthy.
+        </div>
+      </div>
 
       <div class="row">
         <div class="col-sm-4 sm-label-right">
@@ -46,6 +61,10 @@ <h3>Rollback {{serverGroup.name}}</h3>
               <strong>max</strong>: {{ serverGroup.capacity.max }},
               <strong>desired</strong>: {{ serverGroup.capacity.desired }}
             ]<br/>(minimum capacity pinned at {{serverGroup.capacity.desired}} to prevent autoscaling down during rollback)
+              <li ng-if="command.rollbackContext.targetHealthyRollbackPercentage < 100">
+                Wait for at least {{minHealthy(command.rollbackContext.targetHealthyRollbackPercentage)}}
+                instances to report as healthy
+              </li>
             </li>
             <li>Disable {{ serverGroup.name }}</li>
             <li>Restore minimum capacity of <em>{{ command.rollbackContext.restoreServerGroupName || 'previous server group' }}</em> [