Merge pull request #6907 from jackfrancis/nodegroupset-cloudprovider

CAS: cloudprovider-specific nodegroupset

cluster-autoscaler/processors/nodegroupset/aws_nodegroups.go → cluster-autoscaler/cloudprovider/aws/aws_nodegroups.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,17 +14,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package nodegroupset
+package aws
 
 import (
 	"k8s.io/autoscaler/cluster-autoscaler/config"
+	"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 )
 
-// CreateAwsNodeInfoComparator returns a comparator that checks if two nodes should be considered
+// CreateNodeInfoComparator returns a comparator that checks if two nodes should be considered
 // part of the same NodeGroupSet. This is true if they match usual conditions checked by IsCloudProviderNodeInfoSimilar,
 // even if they have different AWS-specific labels.
-func CreateAwsNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.NodeGroupDifferenceRatios) NodeInfoComparator {
+func CreateNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.NodeGroupDifferenceRatios) nodegroupset.NodeInfoComparator {
 	awsIgnoredLabels := map[string]bool{
 		"alpha.eksctl.io/instance-id":    true, // this is a label used by eksctl to identify instances.
 		"alpha.eksctl.io/nodegroup-name": true, // this is a label used by eksctl to identify "node group" names.
@@ -34,7 +35,7 @@ func CreateAwsNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.N
 		"topology.ebs.csi.aws.com/zone":  true, // this is a label used by the AWS EBS CSI driver as a target for Persistent Volume Node Affinity
 	}
 
-	for k, v := range BasicIgnoredLabels {
+	for k, v := range nodegroupset.BasicIgnoredLabels {
 		awsIgnoredLabels[k] = v
 	}
 
@@ -43,6 +44,6 @@ func CreateAwsNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.N
 	}
 
 	return func(n1, n2 *schedulerframework.NodeInfo) bool {
-		return IsCloudProviderNodeInfoSimilar(n1, n2, awsIgnoredLabels, ratioOpts)
+		return nodegroupset.IsCloudProviderNodeInfoSimilar(n1, n2, awsIgnoredLabels, ratioOpts)
 	}
 }

cluster-autoscaler/processors/nodegroupset/aws_nodegroups_test.go → cluster-autoscaler/cloudprovider/aws/aws_nodegroups_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2021 The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,18 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package nodegroupset
+package aws
 
 import (
 	"testing"
 
 	"k8s.io/autoscaler/cluster-autoscaler/config"
 	"k8s.io/autoscaler/cluster-autoscaler/context"
+	"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
 	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
 )
 
 func TestIsAwsNodeInfoSimilar(t *testing.T) {
-	comparator := CreateAwsNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})
+	comparator := CreateNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})
 	node1 := BuildTestNode("node1", 1000, 2000)
 	node2 := BuildTestNode("node2", 1000, 2000)
 
@@ -169,14 +170,14 @@ func TestIsAwsNodeInfoSimilar(t *testing.T) {
 			if tc.removeOneLabel {
 				delete(node2.ObjectMeta.Labels, tc.label)
 			}
-			checkNodesSimilar(t, node1, node2, comparator, true)
+			nodegroupset.CheckNodesSimilar(t, node1, node2, comparator, true)
 		})
 	}
 }
 
 func TestFindSimilarNodeGroupsAwsBasic(t *testing.T) {
 	context := &context.AutoscalingContext{}
-	ni1, ni2, ni3 := buildBasicNodeGroups(context)
-	processor := &BalancingNodeGroupSetProcessor{Comparator: CreateAwsNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})}
-	basicSimilarNodeGroupsTest(t, context, processor, ni1, ni2, ni3)
+	ni1, ni2, ni3 := nodegroupset.BuildBasicNodeGroups(context)
+	processor := &nodegroupset.BalancingNodeGroupSetProcessor{Comparator: CreateNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})}
+	nodegroupset.BasicSimilarNodeGroupsTest(t, context, processor, ni1, ni2, ni3)
 }

cluster-autoscaler/processors/nodegroupset/azure_nodegroups.go → cluster-autoscaler/cloudprovider/azure/azure_nodegroups.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,10 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package nodegroupset
+package azure
 
 import (
 	"k8s.io/autoscaler/cluster-autoscaler/config"
+	"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 )
 
@@ -52,12 +53,12 @@ func nodesFromSameAzureNodePoolLegacy(n1, n2 *schedulerframework.NodeInfo) bool
 	return n1AzureNodePool != "" && n1AzureNodePool == n2AzureNodePool
 }
 
-// CreateAzureNodeInfoComparator returns a comparator that checks if two nodes should be considered
+// CreateNodeInfoComparator returns a comparator that checks if two nodes should be considered
 // part of the same NodeGroupSet. This is true if they either belong to the same Azure agentpool
 // or match usual conditions checked by IsCloudProviderNodeInfoSimilar, even if they have different agentpool labels.
-func CreateAzureNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.NodeGroupDifferenceRatios) NodeInfoComparator {
+func CreateNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.NodeGroupDifferenceRatios) nodegroupset.NodeInfoComparator {
 	azureIgnoredLabels := make(map[string]bool)
-	for k, v := range BasicIgnoredLabels {
+	for k, v := range nodegroupset.BasicIgnoredLabels {
 		azureIgnoredLabels[k] = v
 	}
 	azureIgnoredLabels[AzureNodepoolLegacyLabel] = true
@@ -78,6 +79,6 @@ func CreateAzureNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config
 		if nodesFromSameAzureNodePool(n1, n2) {
 			return true
 		}
-		return IsCloudProviderNodeInfoSimilar(n1, n2, azureIgnoredLabels, ratioOpts)
+		return nodegroupset.IsCloudProviderNodeInfoSimilar(n1, n2, azureIgnoredLabels, ratioOpts)
 	}
 }

cluster-autoscaler/processors/nodegroupset/azure_nodegroups_test.go → cluster-autoscaler/cloudprovider/azure/azure_nodegroups_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package nodegroupset
+package azure
 
 import (
 	"testing"
@@ -23,82 +23,83 @@ import (
 	testprovider "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/test"
 	"k8s.io/autoscaler/cluster-autoscaler/config"
 	"k8s.io/autoscaler/cluster-autoscaler/context"
+	"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
 	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestIsAzureNodeInfoSimilar(t *testing.T) {
-	comparator := CreateAzureNodeInfoComparator([]string{"example.com/ready"}, config.NodeGroupDifferenceRatios{})
+	comparator := CreateNodeInfoComparator([]string{"example.com/ready"}, config.NodeGroupDifferenceRatios{})
 	n1 := BuildTestNode("node1", 1000, 2000)
 	n1.ObjectMeta.Labels["test-label"] = "test-value"
 	n1.ObjectMeta.Labels["character"] = "thing"
 	n2 := BuildTestNode("node2", 1000, 2000)
 	n2.ObjectMeta.Labels["test-label"] = "test-value"
 	// No node-pool labels.
-	checkNodesSimilar(t, n1, n2, comparator, false)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, false)
 	// Empty agentpool labels
 	n1.ObjectMeta.Labels["agentpool"] = ""
 	n2.ObjectMeta.Labels["agentpool"] = ""
-	checkNodesSimilar(t, n1, n2, comparator, false)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, false)
 	// AKS agentpool labels
 	n1.ObjectMeta.Labels["kubernetes.azure.com/agentpool"] = "foo"
 	n2.ObjectMeta.Labels["kubernetes.azure.com/agentpool"] = "bar"
-	checkNodesSimilar(t, n1, n2, comparator, false)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, false)
 	// Only one non empty
 	n1.ObjectMeta.Labels["agentpool"] = ""
 	n2.ObjectMeta.Labels["agentpool"] = "foo"
-	checkNodesSimilar(t, n1, n2, comparator, false)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, false)
 	// Only one present
 	delete(n1.ObjectMeta.Labels, "agentpool")
 	n2.ObjectMeta.Labels["agentpool"] = "foo"
-	checkNodesSimilar(t, n1, n2, comparator, false)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, false)
 	// Different vales
 	n1.ObjectMeta.Labels["agentpool"] = "foo1"
 	n2.ObjectMeta.Labels["agentpool"] = "foo2"
-	checkNodesSimilar(t, n1, n2, comparator, false)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, false)
 	// Same values
 	n1.ObjectMeta.Labels["agentpool"] = "foo"
 	n2.ObjectMeta.Labels["agentpool"] = "foo"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// Same labels except for agentpool
 	delete(n1.ObjectMeta.Labels, "character")
 	n1.ObjectMeta.Labels["agentpool"] = "foo"
 	n2.ObjectMeta.Labels["agentpool"] = "bar"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// Different creationSource
 	n1.ObjectMeta.Labels["creationSource"] = "aks-aks-nodepool2-vmss"
 	n2.ObjectMeta.Labels["creationSource"] = "aks-aks-nodepool3-vmss"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// Different node image version
 	n1.ObjectMeta.Labels["kubernetes.azure.com/node-image-version"] = "AKSUbuntu-1804gen2-2021.01.28"
 	n2.ObjectMeta.Labels["kubernetes.azure.com/node-image-version"] = "AKSUbuntu-1804gen2-2022.01.30"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// Custom label
 	n1.ObjectMeta.Labels["example.com/ready"] = "true"
 	n2.ObjectMeta.Labels["example.com/ready"] = "false"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// One node with aksConsolidatedAdditionalProperties label
 	n1.ObjectMeta.Labels[aksConsolidatedAdditionalProperties] = "foo"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// Same aksConsolidatedAdditionalProperties
 	n2.ObjectMeta.Labels[aksConsolidatedAdditionalProperties] = "foo"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 	// Different aksConsolidatedAdditionalProperties label
 	n2.ObjectMeta.Labels[aksConsolidatedAdditionalProperties] = "bar"
-	checkNodesSimilar(t, n1, n2, comparator, true)
+	nodegroupset.CheckNodesSimilar(t, n1, n2, comparator, true)
 }
 
 func TestFindSimilarNodeGroupsAzureBasic(t *testing.T) {
 	context := &context.AutoscalingContext{}
-	ni1, ni2, ni3 := buildBasicNodeGroups(context)
-	processor := &BalancingNodeGroupSetProcessor{Comparator: CreateAzureNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})}
-	basicSimilarNodeGroupsTest(t, context, processor, ni1, ni2, ni3)
+	ni1, ni2, ni3 := nodegroupset.BuildBasicNodeGroups(context)
+	processor := &nodegroupset.BalancingNodeGroupSetProcessor{Comparator: CreateNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})}
+	nodegroupset.BasicSimilarNodeGroupsTest(t, context, processor, ni1, ni2, ni3)
 }
 
 func TestFindSimilarNodeGroupsAzureByLabel(t *testing.T) {
-	processor := &BalancingNodeGroupSetProcessor{Comparator: CreateAzureNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})}
+	processor := &nodegroupset.BalancingNodeGroupSetProcessor{Comparator: CreateNodeInfoComparator([]string{}, config.NodeGroupDifferenceRatios{})}
 	context := &context.AutoscalingContext{}
 
 	n1 := BuildTestNode("n1", 1000, 1000)

cluster-autoscaler/processors/nodegroupset/gce_nodegroups.go → cluster-autoscaler/cloudprovider/gce/gce_nodegroups.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Kubernetes Authors.
+Copyright 2024 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,22 +14,23 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package nodegroupset
+package gce
 
 import (
 	"k8s.io/autoscaler/cluster-autoscaler/config"
+	"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 )
 
 // CreateGceNodeInfoComparator returns a comparator that checks if two nodes should be considered
 // part of the same NodeGroupSet. This is true if they match usual conditions checked by IsCloudProviderNodeInfoSimilar,
 // even if they have different GCE-specific labels.
-func CreateGceNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.NodeGroupDifferenceRatios) NodeInfoComparator {
+func CreateGceNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.NodeGroupDifferenceRatios) nodegroupset.NodeInfoComparator {
 	gceIgnoredLabels := map[string]bool{
 		"topology.gke.io/zone": true,
 	}
 
-	for k, v := range BasicIgnoredLabels {
+	for k, v := range nodegroupset.BasicIgnoredLabels {
 		gceIgnoredLabels[k] = v
 	}
 
@@ -38,6 +39,6 @@ func CreateGceNodeInfoComparator(extraIgnoredLabels []string, ratioOpts config.N
 	}
 
 	return func(n1, n2 *schedulerframework.NodeInfo) bool {
-		return IsCloudProviderNodeInfoSimilar(n1, n2, gceIgnoredLabels, ratioOpts)
+		return nodegroupset.IsCloudProviderNodeInfoSimilar(n1, n2, gceIgnoredLabels, ratioOpts)
 	}
 }

cluster-autoscaler/main.go

@@ -47,7 +47,10 @@ import (
 	"k8s.io/apiserver/pkg/server/routes"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/azure"
 	cloudBuilder "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/builder"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/gce"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/gce/localssdsize"
 	"k8s.io/autoscaler/cluster-autoscaler/config"
 	"k8s.io/autoscaler/cluster-autoscaler/core"
@@ -568,12 +571,12 @@ func buildAutoscaler(debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter
 	} else {
 		nodeInfoComparatorBuilder := nodegroupset.CreateGenericNodeInfoComparator
 		if autoscalingOptions.CloudProviderName == cloudprovider.AzureProviderName {
-			nodeInfoComparatorBuilder = nodegroupset.CreateAzureNodeInfoComparator
+			nodeInfoComparatorBuilder = azure.CreateNodeInfoComparator
 		} else if autoscalingOptions.CloudProviderName == cloudprovider.AwsProviderName {
-			nodeInfoComparatorBuilder = nodegroupset.CreateAwsNodeInfoComparator
+			nodeInfoComparatorBuilder = aws.CreateNodeInfoComparator
 			opts.Processors.TemplateNodeInfoProvider = nodeinfosprovider.NewAsgTagResourceNodeInfoProvider(nodeInfoCacheExpireTime, *forceDaemonSets)
 		} else if autoscalingOptions.CloudProviderName == cloudprovider.GceProviderName {
-			nodeInfoComparatorBuilder = nodegroupset.CreateGceNodeInfoComparator
+			nodeInfoComparatorBuilder = gce.CreateGceNodeInfoComparator
 			opts.Processors.TemplateNodeInfoProvider = nodeinfosprovider.NewAnnotationNodeInfoProvider(nodeInfoCacheExpireTime, *forceDaemonSets)
 		}
 		nodeInfoComparator = nodeInfoComparatorBuilder(autoscalingOptions.BalancingExtraIgnoredLabels, autoscalingOptions.NodeGroupSetRatios)