test(e2e): validate that pulling from an acr registry attached to aks…

… via --attach-acr with karpenter nodes works (#457)

* test(e2e): validate that pulling from an acr registry attached to aks via --attach-acr with karpenter nodes works

* fix: ci

* test: ci

* fix: crd validation breaks on local so accidentally committed the change with it disabled

* fix: passing in azure acr name from env rather than using makefile default

* fix: ci again?

* fix: nit comments addressed

* test: only provisioning one pod

.github/actions/e2e/create-acr/action.yaml

@@ -40,3 +40,7 @@ runs:
   - name: create ACR
     shell: bash
     run: AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} AZURE_LOCATION=${{ inputs.location }} make az-mkacr
+  - name: import needed images 
+    shell: bash 
+    run: | 
+      AZURE_ACR_NAME=${{ inputs.acr_name }} make az-acrimport

.github/workflows/e2e-matrix.yaml

@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [Nonbehavioral, Utilization, GPU, Drift, Integration, NodeClaim, Chaos]
+        suite: [Nonbehavioral, Utilization, GPU, Drift, Integration, NodeClaim, Chaos, ACR]
     permissions:
       contents: read
       id-token: write

.github/workflows/e2e.yaml

@@ -129,7 +129,7 @@ jobs:
         if: inputs.suite != 'Nonbehavioral'
         run: |
           AZURE_CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_RESOURCE_GROUP=${{ env.RG_NAME }} make az-creds
-          CLUSTER_NAME=${{ env.CLUSTER_NAME }} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests
+          CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_ACR_NAME=${{ env.ACR_NAME}} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests
       - name: dump logs on failure
         uses: ./.github/actions/e2e/dump-logs
         if: failure() || cancelled()

Makefile

@@ -45,7 +45,7 @@ e2etests: ## Run the e2e suite against your local cluster
 	# -count 1: prevents caching
 	# -timeout: If a test binary runs longer than TEST_TIMEOUT, panic
 	# -v: verbose output
-	cd test && CLUSTER_NAME=${CLUSTER_NAME} go test \
+	cd test && CLUSTER_NAME=${CLUSTER_NAME} AZURE_ACR_NAME=${AZURE_ACR_NAME} go test \
 		-p 1 \
 		-count 1 \
 		-timeout ${TEST_TIMEOUT} \

Makefile-az.mk

@@ -38,6 +38,15 @@ az-mkacr: az-mkrg ## Create test ACR
 		--sku Basic --admin-enabled -o none
 	az acr login  --name $(AZURE_ACR_NAME)
 
+az-acrimport: ## Imports an image to an acr registry
+	az acr import --name $(AZURE_ACR_NAME) --source "mcr.microsoft.com/oss/kubernetes/pause:3.6" --image "pause:3.6"
+
+az-cleanenv: az-rmnodeclaims-fin  ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses) 
+	kubectl delete pods -n default --all
+	kubectl delete nodeclaims --all 
+	kubectl delete nodepools --all
+	kubectl delete aksnodeclasses --all
+
 az-mkaks: az-mkacr ## Create test AKS cluster (with --vm-set-type AvailabilitySet for compatibility with standalone VMs)
 	az aks create          --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) --location $(AZURE_LOCATION) \
 		--enable-managed-identity --node-count 3 --generate-ssh-keys --vm-set-type AvailabilitySet -o none

test/suites/acr/suite_test.go

@@ -0,0 +1,80 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package acr
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	"github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/labels"
+	corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
+	"sigs.k8s.io/karpenter/pkg/test"
+)
+
+var env *azure.Environment
+var nodeClass *v1alpha2.AKSNodeClass
+var nodePool *corev1beta1.NodePool
+var pauseImage string
+
+func TestAcr(t *testing.T) {
+	RegisterFailHandler(Fail)
+	BeforeSuite(func() {
+		env = azure.NewEnvironment(t)
+		acrName := os.Getenv("AZURE_ACR_NAME")
+		Expect(acrName).NotTo(BeEmpty(), "AZURE_ACR_NAME must be set for the acr test suite")
+		pauseImage = fmt.Sprintf("%s.azurecr.io/pause:3.6", acrName)
+	})
+	RunSpecs(t, "Acr")
+}
+
+var _ = BeforeEach(func() {
+	env.BeforeEach()
+	nodeClass = env.DefaultAKSNodeClass()
+	nodePool = env.DefaultNodePool(nodeClass)
+})
+var _ = AfterEach(func() { env.Cleanup() })
+var _ = AfterEach(func() { env.AfterEach() })
+
+var _ = Describe("Acr", func() {
+	Describe("Image Pull", func() {
+		It("should allow karpenter user pool nodes to pull images from the clusters attached acr", func() {
+			deployment := test.Deployment(test.DeploymentOptions{
+				Replicas: 1,
+				PodOptions: test.PodOptions{
+					ResourceRequirements: v1.ResourceRequirements{
+						Requests: v1.ResourceList{
+							v1.ResourceCPU: resource.MustParse("1.1"),
+						},
+					},
+					Image: pauseImage,
+				},
+			})
+
+			env.ExpectCreated(nodePool, nodeClass, deployment)
+			env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas))
+		})
+	})
+})