apps: deployment config stuck in the new state should respect timeout…

…Secods

pkg/apps/apis/apps/types.go

@@ -113,6 +113,7 @@ const (
 	DeploymentCancelledNewerDeploymentExists  = "newer deployment was found running"
 	DeploymentFailedUnrelatedDeploymentExists = "unrelated pod with the same name as this deployment is already running"
 	DeploymentFailedDeployerPodNoLongerExists = "deployer pod no longer exists"
+	DeploymentFailedUnableToCreateDeployerPod = "unable to create deployer pod"
 )
 
 // DeploymentStatus describes the possible states a deployment can be in.

pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go

@@ -128,6 +128,18 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
 			return err
 		}
 	}
+
+	// In case the deployment is stuck in "new" state because we fail to create
+	// deployer pod (quota, etc..) we should respect the timeoutSeconds in the
+	// config strategy and transition the rollout to failed instead of waiting for
+	// the deployment pod forever.
+	if latestIsDeployed && deployutil.ConfigExceededTimeoutSeconds(config, latestDeployment) {
+		if err := c.failRunningRollout(latestDeployment); err != nil {
+			return err
+		}
+		return c.updateStatus(config, existingDeployments)
+	}
+
 	// Process triggers and start an initial rollouts
 	configCopy, err := deployutil.DeploymentConfigDeepCopy(config)
 	if err != nil {
@@ -336,6 +348,31 @@ func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentCo
 	return nil
 }
 
+// failRunningRollout updates the phase of the rollout to failed. This is needed
+// when the deployer controller fail to create new deployer pod (because of
+// quota, etc...) which in that case the deployment phase will never transition
+// from new to pending.
+func (c *DeploymentConfigController) failRunningRollout(deployment *v1.ReplicationController) error {
+	// Only allow to update new deployments
+	if !deployutil.IsNewDeployment(deployment) {
+		return nil
+	}
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		rc, err := c.rcLister.ReplicationControllers(deployment.Namespace).Get(deployment.Name)
+		if err != nil {
+			return err
+		}
+		copied, err := deployutil.DeploymentDeepCopyV1(rc)
+		if err != nil {
+			return err
+		}
+		copied.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusFailed)
+		copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnableToCreateDeployerPod
+		_, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied)
+		return err
+	})
+}
+
 // cancelRunningRollouts cancels existing rollouts when the latest deployment does not
 // exists yet to allow new rollout superceded by the new config version.
 func (c *DeploymentConfigController) cancelRunningRollouts(config *deployapi.DeploymentConfig, existingDeployments []*v1.ReplicationController, cm *RCControllerRefManager) error {
@@ -452,6 +489,15 @@ func updateConditions(config *deployapi.DeploymentConfig, newStatus *deployapi.D
 	// Condition about progress.
 	if latestRC != nil {
 		switch deployutil.DeploymentStatusFor(latestRC) {
+		case deployapi.DeploymentStatusNew:
+			// In case we fail to created the deployer pod within strategy
+			// timeoutSeconds do not leave the deployment config in New state forever
+			// but timeout.
+			if deployutil.ConfigExceededTimeoutSeconds(config, latestRC) {
+				msg := fmt.Sprintf("rollout of replication controller %s timed out to create the deployer pod", latestRC.Name)
+				condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionFalse, deployapi.TimedOutReason, msg)
+				deployutil.SetDeploymentCondition(newStatus, *condition)
+			}
 		case deployapi.DeploymentStatusPending:
 			msg := fmt.Sprintf("replication controller %q is waiting for pod %q to run", latestRC.Name, deployutil.DeployerPodNameForDeployment(latestRC.Name))
 			condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionUnknown, "", msg)

pkg/apps/util/util.go

@@ -652,6 +652,12 @@ func IsTerminatedDeployment(deployment runtime.Object) bool {
 	return IsCompleteDeployment(deployment) || IsFailedDeployment(deployment)
 }
 
+// IsNewDeployment returns true if the passed deployment is in new state.
+func IsNewDeployment(deployment runtime.Object) bool {
+	current := DeploymentStatusFor(deployment)
+	return current == deployapi.DeploymentStatusNew
+}
+
 // IsCompleteDeployment returns true if the passed deployment is in state complete.
 func IsCompleteDeployment(deployment runtime.Object) bool {
 	current := DeploymentStatusFor(deployment)
@@ -782,6 +788,23 @@ func DeploymentsForCleanup(configuration *deployapi.DeploymentConfig, deployment
 	return relevantDeployments
 }
 
+func ConfigExceededTimeoutSeconds(config *deployapi.DeploymentConfig, latestRC *v1.ReplicationController) bool {
+	var timeoutSeconds int64
+	if params := config.Spec.Strategy.RollingParams; params != nil {
+		timeoutSeconds = deployapi.DefaultRollingTimeoutSeconds
+		if params.TimeoutSeconds != nil {
+			timeoutSeconds = *params.TimeoutSeconds
+		}
+	}
+	if params := config.Spec.Strategy.RecreateParams; params != nil {
+		timeoutSeconds = deployapi.DefaultRecreateTimeoutSeconds
+		if params.TimeoutSeconds != nil {
+			timeoutSeconds = *params.TimeoutSeconds
+		}
+	}
+	return int64(time.Since(latestRC.CreationTimestamp.Time)*time.Second) > timeoutSeconds
+}
+
 // WaitForRunningDeployerPod waits a given period of time until the deployer pod
 // for given replication controller is not running.
 func WaitForRunningDeployerPod(podClient kcoreclient.PodsGetter, rc *api.ReplicationController, timeout time.Duration) error {