From f06e1798ae493c39b91ae8e17c4289489973c10f Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Mon, 23 Oct 2017 16:59:52 +0200 Subject: [PATCH] apps: deployment config stuck in the new state should respect timeoutSecods --- pkg/apps/apis/apps/types.go | 1 + .../deployer/deployer_controller.go | 15 ++++++++ .../deploymentconfig_controller.go | 1 + pkg/apps/util/util.go | 37 +++++++++++++++++++ pkg/apps/util/util_test.go | 19 ++++++++++ 5 files changed, 73 insertions(+) diff --git a/pkg/apps/apis/apps/types.go b/pkg/apps/apis/apps/types.go index 58338278ce2e..51d426683c4f 100644 --- a/pkg/apps/apis/apps/types.go +++ b/pkg/apps/apis/apps/types.go @@ -113,6 +113,7 @@ const ( DeploymentCancelledNewerDeploymentExists = "newer deployment was found running" DeploymentFailedUnrelatedDeploymentExists = "unrelated pod with the same name as this deployment is already running" DeploymentFailedDeployerPodNoLongerExists = "deployer pod no longer exists" + DeploymentFailedUnableToCreateDeployerPod = "unable to create deployer pod" ) // DeploymentStatus describes the possible states a deployment can be in. diff --git a/pkg/apps/controller/deployer/deployer_controller.go b/pkg/apps/controller/deployer/deployer_controller.go index 349def5be22b..6a9a8b960e8a 100755 --- a/pkg/apps/controller/deployer/deployer_controller.go +++ b/pkg/apps/controller/deployer/deployer_controller.go @@ -127,6 +127,21 @@ func (c *DeploymentController) handle(deployment *v1.ReplicationController, will } break } + // In case the deployment is stuck in "new" state because we fail to create + // deployer pod (quota, etc..) we should respect the timeoutSeconds in the + // config strategy and transition the rollout to failed instead of waiting for + // the deployment pod forever. + config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec) + if err != nil { + return err + } + if deployutil.RolloutExceededTimeoutSeconds(config, deployment) { + nextStatus = deployapi.DeploymentStatusFailed + updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnableToCreateDeployerPod + c.emitDeploymentEvent(deployment, v1.EventTypeWarning, "RolloutTimeout", fmt.Sprintf("Rollout for %q failed to create deployer pod (timeoutSeconds: %d", deployutil.LabelForDeploymentV1(deployment), deployutil.GetTimeoutSecondsForStrategy(config))) + glog.V(4).Infof("Failing deployment %s/%s as we timeout out while waiting for the deployer pod to be created", deployment.Namespace, deployment.Name) + break + } switch { case kerrors.IsNotFound(deployerErr): diff --git a/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go b/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go index 5b44c66a05d7..f11db34ef681 100644 --- a/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go +++ b/pkg/apps/controller/deploymentconfig/deploymentconfig_controller.go @@ -128,6 +128,7 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) return err } } + // Process triggers and start an initial rollouts configCopy, err := deployutil.DeploymentConfigDeepCopy(config) if err != nil { diff --git a/pkg/apps/util/util.go b/pkg/apps/util/util.go index 7fdc5da0ce09..97ecd28a4b61 100644 --- a/pkg/apps/util/util.go +++ b/pkg/apps/util/util.go @@ -652,6 +652,12 @@ func IsTerminatedDeployment(deployment runtime.Object) bool { return IsCompleteDeployment(deployment) || IsFailedDeployment(deployment) } +// IsNewDeployment returns true if the passed deployment is in new state. +func IsNewDeployment(deployment runtime.Object) bool { + current := DeploymentStatusFor(deployment) + return current == deployapi.DeploymentStatusNew +} + // IsCompleteDeployment returns true if the passed deployment is in state complete. func IsCompleteDeployment(deployment runtime.Object) bool { current := DeploymentStatusFor(deployment) @@ -782,6 +788,37 @@ func DeploymentsForCleanup(configuration *deployapi.DeploymentConfig, deployment return relevantDeployments } +// GetTimeoutSecondsForStrategy returns the timeout in seconds defined in the +// deployment config strategy. +func GetTimeoutSecondsForStrategy(config *deployapi.DeploymentConfig) int64 { + var timeoutSeconds int64 + switch config.Spec.Strategy.Type { + case deployapi.DeploymentStrategyTypeRolling: + timeoutSeconds = deployapi.DefaultRollingTimeoutSeconds + if t := config.Spec.Strategy.RollingParams.TimeoutSeconds; t != nil { + timeoutSeconds = *t + } + case deployapi.DeploymentStrategyTypeRecreate: + timeoutSeconds = deployapi.DefaultRecreateTimeoutSeconds + if t := config.Spec.Strategy.RecreateParams.TimeoutSeconds; t != nil { + timeoutSeconds = *t + } + case deployapi.DeploymentStrategyTypeCustom: + timeoutSeconds = deployapi.DefaultRecreateTimeoutSeconds + } + return timeoutSeconds +} + +// RolloutExceededTimeoutSeconds returns true if the current deployment exceeded +// the timeoutSeconds defined for its strategy. +// Note that this is different than activeDeadlineSeconds which is the timeout +// set for the deployer pod. In some cases, the deployer pod cannot be created +// (like quota, etc...). In that case deployer controller use this function to +// measure if the created deployment (RC) exceeded the timeout. +func RolloutExceededTimeoutSeconds(config *deployapi.DeploymentConfig, latestRC *v1.ReplicationController) bool { + return int64(time.Since(latestRC.CreationTimestamp.Time).Seconds()) > GetTimeoutSecondsForStrategy(config) +} + // WaitForRunningDeployerPod waits a given period of time until the deployer pod // for given replication controller is not running. func WaitForRunningDeployerPod(podClient kcoreclient.PodsGetter, rc *api.ReplicationController, timeout time.Duration) error { diff --git a/pkg/apps/util/util_test.go b/pkg/apps/util/util_test.go index 34d4fdfc6882..15405a7d1a59 100644 --- a/pkg/apps/util/util_test.go +++ b/pkg/apps/util/util_test.go @@ -586,3 +586,22 @@ func TestRemoveCondition(t *testing.T) { } } } + +func TestRolloutExceededTimeoutSeconds(t *testing.T) { + config := deploytest.OkDeploymentConfig(1) + timeoutSeconds := int64(10) + config.Spec.Strategy.RecreateParams.TimeoutSeconds = &timeoutSeconds + deployment, err := MakeDeploymentV1(config, kapi.Codecs.LegacyCodec(deployv1.SchemeGroupVersion)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + deployment.ObjectMeta.CreationTimestamp = metav1.Time{Time: time.Now().Add(-20 * time.Second)} + if !RolloutExceededTimeoutSeconds(config, deployment) { + t.Errorf("expected rollout timeout") + } + deployment.ObjectMeta.CreationTimestamp = metav1.Time{Time: time.Now().Add(-5 * time.Second)} + if RolloutExceededTimeoutSeconds(config, deployment) { + t.Errorf("unexpected rollout timeout") + } + +}