Merge pull request #6994 from nojnhuh/azure-e2e-test

cluster-autoscaler Azure e2e: move Helm to ginkgo
kubernetes · Jul 3, 2024 · bb5d064 · bb5d064
2 parents 4d05113 + 3f18f06
commit bb5d064
Show file tree

Hide file tree

Showing 5 changed files with 515 additions and 71 deletions.
diff --git a/cluster-autoscaler/cloudprovider/azure/test/Makefile b/cluster-autoscaler/cloudprovider/azure/test/Makefile
@@ -1,62 +1,25 @@
 REPO_ROOT:=$(shell git rev-parse --show-toplevel)
 CAS_ROOT:=$(REPO_ROOT)/cluster-autoscaler
-CAS_CHART:=$(REPO_ROOT)/charts/cluster-autoscaler
 
 include $(CAS_ROOT)/Makefile
 
-TOOLS_BIN_DIR := $(abspath bin)
-
-export PATH := $(TOOLS_BIN_DIR):$(PATH)
-
-HELM_VER := v3.15.2
-HELM_BIN := helm
-HELM := $(TOOLS_BIN_DIR)/$(HELM_BIN)-$(HELM_VER)
-
-helm: $(HELM)
-
-$(HELM):
-	mkdir -p $(TOOLS_BIN_DIR)
-	rm -f "$(TOOLS_BIN_DIR)/$(HELM_BIN)*"
-	curl --retry 3 -fsSL -o $(TOOLS_BIN_DIR)/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
-	chmod 700 $(TOOLS_BIN_DIR)/get_helm.sh
-	USE_SUDO=false HELM_INSTALL_DIR=$(TOOLS_BIN_DIR) DESIRED_VERSION=$(HELM_VER) BINARY_NAME=$(HELM_BIN)-$(HELM_VER) $(TOOLS_BIN_DIR)/get_helm.sh
-	ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
-	rm -f $(TOOLS_BIN_DIR)/get_helm.sh
-
 CLUSTER_AUTOSCALER_NAMESPACE?=default
 CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME?=cluster-autoscaler
 
-.PHONY: install-e2e
-install-e2e: $(HELM)
+.PHONY: build-e2e
+build-e2e:
 	$(MAKE) -C $(CAS_ROOT) build-arch-$(GOARCH) make-image-arch-$(GOARCH)
 	docker push $(IMAGE)-$(GOARCH):$(TAG)
-	$(HELM) upgrade --install cluster-autoscaler $(CAS_CHART) \
-		--namespace $(CLUSTER_AUTOSCALER_NAMESPACE) --create-namespace \
-		--set cloudProvider=azure \
-		--set azureTenantID=$(AZURE_TENANT_ID) \
-		--set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \
-		--set azureUseWorkloadIdentityExtension=true \
-		--set-string podLabels."azure\.workload\.identity/use"=true \
-		--set rbac.serviceAccount.name=$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME) \
-		--set rbac.serviceAccount.annotations."azure\.workload\.identity/tenant-id"=$(AZURE_TENANT_ID) \
-		--set rbac.serviceAccount.annotations."azure\.workload\.identity/client-id"="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \
-		--set autoDiscovery.clusterName="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \
-		--set azureResourceGroup="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \
-		--set nodeSelector."kubernetes\.io/os"=linux \
-		--set image.repository=$(IMAGE)-$(GOARCH) \
-		--set image.tag=$(TAG) \
-		--set image.pullPolicy=Always \
-		--set extraArgs.scale-down-delay-after-add=10s \
-		--set extraArgs.scale-down-unneeded-time=10s \
-		--set extraArgs.scale-down-candidates-pool-ratio=1.0 \
-		--set extraArgs.unremovable-node-recheck-timeout=10s \
-		--set extraArgs.skip-nodes-with-system-pods=false \
-		--set extraArgs.skip-nodes-with-local-storage=false \
-		--wait
 
 ARTIFACTS?=_artifacts
 
 .PHONY: test-e2e
-test-e2e: install-e2e
+test-e2e: build-e2e
 	go run github.com/onsi/ginkgo/v2/ginkgo -v --trace --output-dir "$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" e2e -- \
-		-resource-group="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')"
+		-resource-group="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \
+		-cluster-name="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \
+		-client-id="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \
+		-cas-namespace="$(CLUSTER_AUTOSCALER_NAMESPACE)" \
+		-cas-serviceaccount-name="$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME)" \
+		-cas-image-repository="$(IMAGE)-$(GOARCH)" \
+		-cas-image-tag="$(TAG)"
diff --git a/cluster-autoscaler/cloudprovider/azure/test/e2e/azure_test.go b/cluster-autoscaler/cloudprovider/azure/test/e2e/azure_test.go
@@ -53,6 +53,17 @@ var _ = Describe("Azure Provider", func() {
 	})
 
 	It("scales up AKS node pools when pending Pods exist", func() {
+		ensureHelmValues(map[string]interface{}{
+			"extraArgs": map[string]interface{}{
+				"scale-down-delay-after-add":       "10s",
+				"scale-down-unneeded-time":         "10s",
+				"scale-down-candidates-pool-ratio": "1.0",
+				"unremovable-node-recheck-timeout": "10s",
+				"skip-nodes-with-system-pods":      "false",
+				"skip-nodes-with-local-storage":    "false",
+			},
+		})
+
 		nodes := &corev1.NodeList{}
 		Expect(k8s.List(ctx, nodes)).To(Succeed())
 		nodeCountBefore := len(nodes.Items)

diff --git a/cluster-autoscaler/cloudprovider/azure/test/e2e/e2e_suite_test.go b/cluster-autoscaler/cloudprovider/azure/test/e2e/e2e_suite_test.go
@@ -18,28 +18,52 @@ package e2e_test
 
 import (
 	"context"
+	"errors"
 	"flag"
+	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/storage/driver"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const (
+	casReleaseName = "cluster-autoscaler"
+)
+
 var (
-	ctx           = context.Background()
-	vmss          *armcompute.VirtualMachineScaleSetsClient
-	k8s           client.Client
-	resourceGroup string
+	ctx     = context.Background()
+	vmss    *armcompute.VirtualMachineScaleSetsClient
+	k8s     client.Client
+	helmEnv = cli.New()
+
+	resourceGroup         string
+	clusterName           string
+	clientID              string
+	casNamespace          string
+	casServiceAccountName string
+	casImageRepository    string
+	casImageTag           string
 )
 
 func init() {
 	flag.StringVar(&resourceGroup, "resource-group", "", "resource group containing cluster-autoscaler-managed resources")
+	flag.StringVar(&clusterName, "cluster-name", "", "Cluster API Cluster name for the cluster to be managed by cluster-autoscaler")
+	flag.StringVar(&clientID, "client-id", "", "Azure client ID to be used by cluster-autoscaler")
+	flag.StringVar(&casNamespace, "cas-namespace", "", "Namespace in which to install cluster-autoscaler")
+	flag.StringVar(&casServiceAccountName, "cas-serviceaccount-name", "", "Name of the ServiceAccount to be used by cluster-autoscaler")
+	flag.StringVar(&casImageRepository, "cas-image-repository", "", "Repository of the container image for cluster-autoscaler")
+	flag.StringVar(&casImageTag, "cas-image-tag", "", "Tag of the container image for cluster-autoscaler")
 }
 
 func TestE2E(t *testing.T) {
@@ -53,11 +77,41 @@ var _ = BeforeSuite(func() {
 	vmss, err = armcompute.NewVirtualMachineScaleSetsClient(os.Getenv("AZURE_SUBSCRIPTION_ID"), azCred, nil)
 	Expect(err).NotTo(HaveOccurred())
 
-	k8sConfig := genericclioptions.NewConfigFlags(false)
-	restConfig, err := k8sConfig.ToRESTConfig()
+	restConfig, err := helmEnv.RESTClientGetter().ToRESTConfig()
 	Expect(err).NotTo(HaveOccurred())
 	k8s, err = client.New(restConfig, client.Options{})
 	Expect(err).NotTo(HaveOccurred())
+
+	ensureHelmValues(map[string]interface{}{
+		"cloudProvider":                     "azure",
+		"azureTenantID":                     os.Getenv("AZURE_TENANT_ID"),
+		"azureSubscriptionID":               os.Getenv("AZURE_SUBSCRIPTION_ID"),
+		"azureUseWorkloadIdentityExtension": true,
+		"azureResourceGroup":                resourceGroup,
+		"podLabels": map[string]interface{}{
+			"azure.workload.identity/use": "true",
+		},
+		"rbac": map[string]interface{}{
+			"serviceAccount": map[string]interface{}{
+				"name": casServiceAccountName,
+				"annotations": map[string]interface{}{
+					"azure.workload.identity/tenant-id": os.Getenv("AZURE_TENANT_ID"),
+					"azure.workload.identity/client-id": clientID,
+				},
+			},
+		},
+		"autoDiscovery": map[string]interface{}{
+			"clusterName": clusterName,
+		},
+		"nodeSelector": map[string]interface{}{
+			"kubernetes.io/os": "linux",
+		},
+		"image": map[string]interface{}{
+			"repository": casImageRepository,
+			"tag":        casImageTag,
+			"pullPolicy": "Always",
+		},
+	})
 })
 
 func allVMSSStable(g Gomega) {
@@ -86,3 +140,36 @@ func allVMSSStable(g Gomega) {
 		})),
 	))
 }
+
+func ensureHelmValues(values map[string]interface{}) {
+	helmCfg := new(action.Configuration)
+	Expect(helmCfg.Init(helmEnv.RESTClientGetter(), casNamespace, "secret", func(format string, v ...interface{}) {
+		GinkgoLogr.Info(fmt.Sprintf(format, v...))
+	})).To(Succeed())
+
+	chart, err := loader.Load("../../../../../charts/cluster-autoscaler")
+	Expect(err).NotTo(HaveOccurred())
+
+	get := action.NewGet(helmCfg)
+	_, err = get.Run(casReleaseName)
+	if errors.Is(err, driver.ErrReleaseNotFound) {
+		install := action.NewInstall(helmCfg)
+		install.Timeout = 5 * time.Minute
+		install.Wait = true
+		install.CreateNamespace = true
+		install.ReleaseName = casReleaseName
+		install.Namespace = casNamespace
+		_, err := install.Run(chart, values)
+		Expect(err).NotTo(HaveOccurred())
+		return
+	} else {
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	upgrade := action.NewUpgrade(helmCfg)
+	upgrade.Timeout = 5 * time.Minute
+	upgrade.Wait = true
+	upgrade.ReuseValues = true
+	_, err = upgrade.Run(casReleaseName, chart, values)
+	Expect(err).NotTo(HaveOccurred())
+}
diff --git a/cluster-autoscaler/cloudprovider/azure/test/go.mod b/cluster-autoscaler/cloudprovider/azure/test/go.mod
@@ -7,28 +7,56 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
+	helm.sh/helm/v3 v3.15.2
 	k8s.io/api v0.30.2
 	k8s.io/apimachinery v0.30.2
-	k8s.io/cli-runtime v0.30.2
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	sigs.k8s.io/controller-runtime v0.18.4
 )
 
 require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/Masterminds/squirrel v1.5.4 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/containerd/containerd v1.7.12 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/cli v25.0.1+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v25.0.5+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -39,25 +67,64 @@ require (
 	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmoiron/sqlx v1.3.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc6 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/spf13/cobra v1.7.0 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rubenv/sql-migrate v1.5.2 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
+	go.opentelemetry.io/otel v1.19.0 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
@@ -69,16 +136,24 @@ require (
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/grpc v1.58.3 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.30.1 // indirect
+	k8s.io/apiserver v0.30.1 // indirect
+	k8s.io/cli-runtime v0.30.2 // indirect
 	k8s.io/client-go v0.30.2 // indirect
+	k8s.io/component-base v0.30.1 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/kubectl v0.30.0 // indirect
+	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )