Merge pull request #6989 from nojnhuh/azure-e2e-test

Add Azure cluster-autoscaler e2e test
kubernetes · Jun 28, 2024 · 069aab7 · 069aab7
2 parents 8dd36c9 + fcba053
commit 069aab7
Show file tree

Hide file tree

Showing 6 changed files with 627 additions and 9 deletions.
diff --git a/cluster-autoscaler/cloudprovider/azure/test/Makefile b/cluster-autoscaler/cloudprovider/azure/test/Makefile
@@ -30,7 +30,7 @@ CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME?=cluster-autoscaler
 install-e2e: $(HELM)
 	$(MAKE) -C $(CAS_ROOT) build-arch-$(GOARCH) make-image-arch-$(GOARCH)
 	docker push $(IMAGE)-$(GOARCH):$(TAG)
-	$(HELM) install cluster-autoscaler $(CAS_CHART) \
+	$(HELM) upgrade --install cluster-autoscaler $(CAS_CHART) \
 		--namespace $(CLUSTER_AUTOSCALER_NAMESPACE) --create-namespace \
 		--set cloudProvider=azure \
 		--set azureTenantID=$(AZURE_TENANT_ID) \
@@ -46,8 +46,15 @@ install-e2e: $(HELM)
 		--set image.repository=$(IMAGE)-$(GOARCH) \
 		--set image.tag=$(TAG) \
 		--set image.pullPolicy=Always \
+		--set extraArgs.scale-down-delay-after-add=10s \
+		--set extraArgs.scale-down-unneeded-time=10s \
+		--set extraArgs.scale-down-candidates-pool-ratio=1.0 \
+		--set extraArgs.unremovable-node-recheck-timeout=10s \
+		--set extraArgs.skip-nodes-with-system-pods=false \
+		--set extraArgs.skip-nodes-with-local-storage=false \
 		--wait
 
 .PHONY: test-e2e
 test-e2e: install-e2e
-	# TODO
+	go run github.com/onsi/ginkgo/v2/ginkgo e2e -v -- \
+		-resource-group="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')"
diff --git a/cluster-autoscaler/cloudprovider/azure/test/e2e/azure_test.go b/cluster-autoscaler/cloudprovider/azure/test/e2e/azure_test.go
@@ -0,0 +1,143 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("Azure Provider", func() {
+	var (
+		namespace *corev1.Namespace
+	)
+
+	BeforeEach(func() {
+		Eventually(allVMSSStable, "10m", "30s").Should(Succeed())
+
+		namespace = &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "azure-e2e-",
+			},
+		}
+		Expect(k8s.Create(ctx, namespace)).To(Succeed())
+	})
+
+	AfterEach(func() {
+		Expect(k8s.Delete(ctx, namespace)).To(Succeed())
+		Eventually(func() bool {
+			err := k8s.Get(ctx, client.ObjectKeyFromObject(namespace), &corev1.Namespace{})
+			GinkgoLogr.Info("got err", "error", err)
+			return apierrors.IsNotFound(err)
+		}, "1m", "5s").Should(BeTrue(), "Namespace "+namespace.Name+" still exists")
+	})
+
+	It("scales up AKS node pools when pending Pods exist", func() {
+		nodes := &corev1.NodeList{}
+		Expect(k8s.List(ctx, nodes)).To(Succeed())
+		nodeCountBefore := len(nodes.Items)
+
+		By("Creating 100 Pods")
+		// https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/application/php-apache.yaml
+		deploy := &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "php-apache",
+				Namespace: namespace.Name,
+			},
+			Spec: appsv1.DeploymentSpec{
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"run": "php-apache",
+					},
+				},
+				Replicas: ptr.To[int32](100),
+				Template: corev1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"run": "php-apache",
+						},
+					},
+					Spec: corev1.PodSpec{
+						Containers: []corev1.Container{
+							{
+								Name:  "php-apache",
+								Image: "registry.k8s.io/hpa-example",
+								Resources: corev1.ResourceRequirements{
+									Limits: corev1.ResourceList{
+										corev1.ResourceCPU: resource.MustParse("500m"),
+									},
+									Requests: corev1.ResourceList{
+										corev1.ResourceCPU: resource.MustParse("200m"),
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+		Expect(k8s.Create(ctx, deploy)).To(Succeed())
+
+		By("Waiting for more Ready Nodes to exist")
+		Eventually(func() (int, error) {
+			readyCount := 0
+			nodes := &corev1.NodeList{}
+			if err := k8s.List(ctx, nodes); err != nil {
+				return 0, err
+			}
+			for _, node := range nodes.Items {
+				for _, cond := range node.Status.Conditions {
+					if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue {
+						readyCount++
+						break
+					}
+				}
+			}
+			return readyCount, nil
+		}, "10m", "10s").Should(BeNumerically(">", nodeCountBefore))
+
+		Eventually(allVMSSStable, "10m", "30s").Should(Succeed())
+
+		By("Deleting 100 Pods")
+		Expect(k8s.Delete(ctx, deploy)).To(Succeed())
+
+		By("Waiting for the original number of Nodes to be Ready")
+		Eventually(func() (int, error) {
+			readyCount := 0
+			nodes := &corev1.NodeList{}
+			if err := k8s.List(ctx, nodes); err != nil {
+				return 0, err
+			}
+			for _, node := range nodes.Items {
+				for _, cond := range node.Status.Conditions {
+					if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue {
+						readyCount++
+						break
+					}
+				}
+			}
+			return readyCount, nil
+		}, "10m", "10s").Should(Equal(nodeCountBefore))
+	})
+})
diff --git a/cluster-autoscaler/cloudprovider/azure/test/e2e/e2e_suite_test.go b/cluster-autoscaler/cloudprovider/azure/test/e2e/e2e_suite_test.go
@@ -0,0 +1,90 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_test
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	ctx           = context.Background()
+	vmss          *armcompute.VirtualMachineScaleSetsClient
+	k8s           client.Client
+	resourceGroup string
+)
+
+func init() {
+	flag.StringVar(&resourceGroup, "resource-group", "", "resource group containing cluster-autoscaler-managed resources")
+}
+
+func TestE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "e2e Suite")
+}
+
+var _ = BeforeSuite(func() {
+	azCred, err := azidentity.NewDefaultAzureCredential(nil)
+	Expect(err).NotTo(HaveOccurred())
+	vmss, err = armcompute.NewVirtualMachineScaleSetsClient(os.Getenv("AZURE_SUBSCRIPTION_ID"), azCred, nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	k8sConfig := genericclioptions.NewConfigFlags(false)
+	restConfig, err := k8sConfig.ToRESTConfig()
+	Expect(err).NotTo(HaveOccurred())
+	k8s, err = client.New(restConfig, client.Options{})
+	Expect(err).NotTo(HaveOccurred())
+})
+
+func allVMSSStable(g Gomega) {
+	pager := vmss.NewListPager(resourceGroup, nil)
+	expectedNodes := 0
+	for pager.More() {
+		page, err := pager.NextPage(ctx)
+		g.Expect(err).NotTo(HaveOccurred())
+		for _, scaleset := range page.Value {
+			g.Expect(*scaleset.Properties.ProvisioningState).To(Equal("Succeeded"))
+			expectedNodes += int(*scaleset.SKU.Capacity)
+		}
+	}
+
+	nodes := &corev1.NodeList{}
+	g.Expect(k8s.List(ctx, nodes)).To(Succeed())
+	g.Expect(nodes.Items).To(SatisfyAll(
+		HaveLen(int(expectedNodes)),
+		ContainElements(Satisfy(func(node corev1.Node) bool {
+			ready := false
+			for _, cond := range node.Status.Conditions {
+				if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue {
+					ready = true
+					break
+				}
+			}
+			return ready
+		})),
+	))
+}
diff --git a/cluster-autoscaler/cloudprovider/azure/test/go.mod b/cluster-autoscaler/cloudprovider/azure/test/go.mod
@@ -0,0 +1,84 @@
+module k8s.io/autoscaler/cluster-autoscaler/cloudprovider/azure/test
+
+go 1.22.3
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0
+	github.com/onsi/ginkgo/v2 v2.19.0
+	github.com/onsi/gomega v1.33.1
+	k8s.io/api v0.30.2
+	k8s.io/apimachinery v0.30.2
+	k8s.io/cli-runtime v0.30.2
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	sigs.k8s.io/controller-runtime v0.18.4
+)
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/cobra v1.7.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/term v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/client-go v0.30.2 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)