diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index a18b1c58fbc1..2dc78b5f6f64 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -289,6 +289,8 @@ type AutoscalingOptions struct { BypassedSchedulers map[string]bool // ProvisioningRequestEnabled tells if CA processes ProvisioningRequest. ProvisioningRequestEnabled bool + // MaxCloudProviderNodeDeletionTime is the maximum time needed by cloud provider to delete a node + MaxCloudProviderNodeDeletionTime time.Duration } // KubeClientOptions specify options for kube client diff --git a/cluster-autoscaler/core/scaledown/actuation/delete_in_batch.go b/cluster-autoscaler/core/scaledown/actuation/delete_in_batch.go index f181b5ae05e8..eb997daf4fc8 100644 --- a/cluster-autoscaler/core/scaledown/actuation/delete_in_batch.go +++ b/cluster-autoscaler/core/scaledown/actuation/delete_in_batch.go @@ -41,8 +41,6 @@ import ( const ( // MaxKubernetesEmptyNodeDeletionTime is the maximum time needed by Kubernetes to delete an empty node. MaxKubernetesEmptyNodeDeletionTime = 3 * time.Minute - // MaxCloudProviderNodeDeletionTime is the maximum time needed by cloud provider to delete a node. - MaxCloudProviderNodeDeletionTime = 5 * time.Minute ) // NodeDeletionBatcher batch scale down candidates for one node group and remove them. @@ -180,9 +178,9 @@ func nodeScaleDownReason(node *apiv1.Node, drain bool) metrics.NodeScaleDownReas } // IsNodeBeingDeleted returns true iff a given node is being deleted. -func IsNodeBeingDeleted(node *apiv1.Node, timestamp time.Time) bool { +func IsNodeBeingDeleted(ctx *context.AutoscalingContext, node *apiv1.Node, timestamp time.Time) bool { deleteTime, _ := taints.GetToBeDeletedTime(node) - return deleteTime != nil && (timestamp.Sub(*deleteTime) < MaxCloudProviderNodeDeletionTime || timestamp.Sub(*deleteTime) < MaxKubernetesEmptyNodeDeletionTime) + return deleteTime != nil && (timestamp.Sub(*deleteTime) < ctx.MaxCloudProviderNodeDeletionTime || timestamp.Sub(*deleteTime) < MaxKubernetesEmptyNodeDeletionTime) } // CleanUpAndRecordFailedScaleDownEvent record failed scale down event and log an error. @@ -203,7 +201,7 @@ func CleanUpAndRecordFailedScaleDownEvent(ctx *context.AutoscalingContext, node func RegisterAndRecordSuccessfulScaleDownEvent(ctx *context.AutoscalingContext, scaleStateNotifier nodegroupchange.NodeGroupChangeObserver, node *apiv1.Node, nodeGroup cloudprovider.NodeGroup, drain bool, nodeDeletionTracker *deletiontracker.NodeDeletionTracker) { ctx.Recorder.Eventf(node, apiv1.EventTypeNormal, "ScaleDown", "nodes removed by cluster autoscaler") currentTime := time.Now() - expectedDeleteTime := time.Now().Add(MaxCloudProviderNodeDeletionTime) + expectedDeleteTime := time.Now().Add(ctx.MaxCloudProviderNodeDeletionTime) scaleStateNotifier.RegisterScaleDown(nodeGroup, node.Name, currentTime, expectedDeleteTime) gpuConfig := ctx.CloudProvider.GetNodeGpuConfig(node) metricResourceName, metricGpuType := gpu.GetGpuInfoForMetrics(gpuConfig, ctx.CloudProvider.GetAvailableGPUTypes(), node, nodeGroup) diff --git a/cluster-autoscaler/core/scaledown/eligibility/eligibility.go b/cluster-autoscaler/core/scaledown/eligibility/eligibility.go index f6cf9a9e55c9..27f18965bff9 100644 --- a/cluster-autoscaler/core/scaledown/eligibility/eligibility.go +++ b/cluster-autoscaler/core/scaledown/eligibility/eligibility.go @@ -109,7 +109,7 @@ func (c *Checker) FilterOutUnremovable(context *context.AutoscalingContext, scal func (c *Checker) unremovableReasonAndNodeUtilization(context *context.AutoscalingContext, timestamp time.Time, nodeInfo *schedulerframework.NodeInfo, utilLogsQuota *klogx.Quota) (simulator.UnremovableReason, *utilization.Info) { node := nodeInfo.Node() - if actuation.IsNodeBeingDeleted(node, timestamp) { + if actuation.IsNodeBeingDeleted(context, node, timestamp) { klog.V(1).Infof("Skipping %s from delete consideration - the node is currently being deleted", node.Name) return simulator.CurrentlyBeingDeleted, nil } diff --git a/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go b/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go index 984504bbe93c..18f3a32753e1 100644 --- a/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go +++ b/cluster-autoscaler/core/scaledown/eligibility/eligibility_test.go @@ -164,6 +164,7 @@ func TestFilterOutUnremovable(t *testing.T) { ScaleDownUnreadyTime: config.DefaultScaleDownUnreadyTime, IgnoreDaemonSetsUtilization: tc.ignoreDaemonSetsUtilization, }, + MaxCloudProviderNodeDeletionTime: 5 * time.Minute, } s := nodegroupconfig.NewDefaultNodeGroupConfigProcessor(options.NodeGroupDefaults) c := NewChecker(s) diff --git a/cluster-autoscaler/core/scaledown/resource/limits.go b/cluster-autoscaler/core/scaledown/resource/limits.go index b87ef0af9d5b..1f92f37f0d9b 100644 --- a/cluster-autoscaler/core/scaledown/resource/limits.go +++ b/cluster-autoscaler/core/scaledown/resource/limits.go @@ -62,7 +62,7 @@ func NoLimits() Limits { // LimitsLeft returns the amount of each resource that can be deleted from the // cluster without violating any constraints. func (lf *LimitsFinder) LimitsLeft(context *context.AutoscalingContext, nodes []*apiv1.Node, resourceLimiter *cloudprovider.ResourceLimiter, timestamp time.Time) Limits { - totalCores, totalMem := coresMemoryTotal(nodes, timestamp) + totalCores, totalMem := coresMemoryTotal(context, nodes, timestamp) var totalResources map[string]int64 var totalResourcesErr error @@ -102,10 +102,10 @@ func computeAboveMin(total int64, min int64) int64 { return 0 } -func coresMemoryTotal(nodes []*apiv1.Node, timestamp time.Time) (int64, int64) { +func coresMemoryTotal(ctx *context.AutoscalingContext, nodes []*apiv1.Node, timestamp time.Time) (int64, int64) { var coresTotal, memoryTotal int64 for _, node := range nodes { - if actuation.IsNodeBeingDeleted(node, timestamp) { + if actuation.IsNodeBeingDeleted(ctx, node, timestamp) { // Nodes being deleted do not count towards total cluster resources continue } @@ -122,7 +122,7 @@ func (lf *LimitsFinder) customResourcesTotal(context *context.AutoscalingContext result := make(map[string]int64) ngCache := make(map[string][]customresources.CustomResourceTarget) for _, node := range nodes { - if actuation.IsNodeBeingDeleted(node, timestamp) { + if actuation.IsNodeBeingDeleted(context, node, timestamp) { // Nodes being deleted do not count towards total cluster resources continue } diff --git a/cluster-autoscaler/core/scaledown/resource/limits_test.go b/cluster-autoscaler/core/scaledown/resource/limits_test.go index e4cb63ed7710..9b6dee2bece6 100644 --- a/cluster-autoscaler/core/scaledown/resource/limits_test.go +++ b/cluster-autoscaler/core/scaledown/resource/limits_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "k8s.io/autoscaler/cluster-autoscaler/config" . "k8s.io/autoscaler/cluster-autoscaler/core/test" "k8s.io/autoscaler/cluster-autoscaler/core/utils" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" @@ -55,7 +56,12 @@ func TestCalculateCoresAndMemoryTotal(t *testing.T) { }, } - coresTotal, memoryTotal := coresMemoryTotal(nodes, time.Now()) + options := config.AutoscalingOptions{ + MaxCloudProviderNodeDeletionTime: 5 * time.Minute, + } + context, err := NewScaleTestAutoscalingContext(options, nil, nil, nil, nil, nil) + assert.NoError(t, err) + coresTotal, memoryTotal := coresMemoryTotal(&context, nodes, time.Now()) assert.Equal(t, int64(42), coresTotal) assert.Equal(t, int64(44000*utils.MiB), memoryTotal) diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index bbbe77c93ac5..c5904059f0bd 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -332,7 +332,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr } // Update cluster resource usage metrics - coresTotal, memoryTotal := calculateCoresMemoryTotal(allNodes, currentTime) + coresTotal, memoryTotal := calculateCoresMemoryTotal(a.AutoscalingContext, allNodes, currentTime) metrics.UpdateClusterCPUCurrentCores(coresTotal) metrics.UpdateClusterMemoryCurrentBytes(memoryTotal) @@ -1052,12 +1052,12 @@ func getUpcomingNodeInfos(upcomingCounts map[string]int, nodeInfos map[string]*s return upcomingNodes } -func calculateCoresMemoryTotal(nodes []*apiv1.Node, timestamp time.Time) (int64, int64) { +func calculateCoresMemoryTotal(context *context.AutoscalingContext, nodes []*apiv1.Node, timestamp time.Time) (int64, int64) { // this function is essentially similar to the calculateScaleDownCoresMemoryTotal // we want to check all nodes, aside from those deleting, to sum the cluster resource usage. var coresTotal, memoryTotal int64 for _, node := range nodes { - if actuation.IsNodeBeingDeleted(node, timestamp) { + if actuation.IsNodeBeingDeleted(context, node, timestamp) { // Nodes being deleted do not count towards total cluster resources continue } diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index f3bd86653855..85d3c9fe7da7 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -257,8 +257,9 @@ var ( "--max-graceful-termination-sec flag should not be set when this flag is set. Not setting this flag will use unordered evictor by default."+ "Priority evictor reuses the concepts of drain logic in kubelet(https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2712-pod-priority-based-graceful-node-shutdown#migration-from-the-node-graceful-shutdown-feature)."+ "Eg. flag usage: '10000:20,1000:100,0:60'") - provisioningRequestsEnabled = flag.Bool("enable-provisioning-requests", false, "Whether the clusterautoscaler will be handling the ProvisioningRequest CRs.") - frequentLoopsEnabled = flag.Bool("frequent-loops-enabled", false, "Whether clusterautoscaler triggers new iterations more frequently when it's needed") + provisioningRequestsEnabled = flag.Bool("enable-provisioning-requests", false, "Whether the clusterautoscaler will be handling the ProvisioningRequest CRs.") + frequentLoopsEnabled = flag.Bool("frequent-loops-enabled", false, "Whether clusterautoscaler triggers new iterations more frequently when it's needed") + maxCloudProviderNodeDeletionTime = flag.Duration("max-cloud-provider-node-deletion-time", 5*time.Minute, "Maximum time needed by cloud provider to delete a node.") ) func isFlagPassed(name string) bool { @@ -432,6 +433,7 @@ func createAutoscalingOptions() config.AutoscalingOptions { DynamicNodeDeleteDelayAfterTaintEnabled: *dynamicNodeDeleteDelayAfterTaintEnabled, BypassedSchedulers: scheduler_util.GetBypassedSchedulersMap(*bypassedSchedulers), ProvisioningRequestEnabled: *provisioningRequestsEnabled, + MaxCloudProviderNodeDeletionTime: *maxCloudProviderNodeDeletionTime, } }