diff --git a/.github/actions/e2e/create-acr/action.yaml b/.github/actions/e2e/create-acr/action.yaml index a485e80ff..aadfed21c 100644 --- a/.github/actions/e2e/create-acr/action.yaml +++ b/.github/actions/e2e/create-acr/action.yaml @@ -40,3 +40,7 @@ runs: - name: create ACR shell: bash run: AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} AZURE_LOCATION=${{ inputs.location }} make az-mkacr + - name: import needed images + shell: bash + run: | + AZURE_ACR_NAME=${{ inputs.acr_name }} make az-acrimport diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index 6dde9d0de..43ce1a351 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -43,7 +43,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [Nonbehavioral, Utilization, GPU, Drift, Integration, NodeClaim, Chaos] + suite: [Nonbehavioral, Utilization, GPU, Drift, Integration, NodeClaim, Chaos, ACR] permissions: contents: read id-token: write diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 548e54cde..f9a0abecd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -129,7 +129,7 @@ jobs: if: inputs.suite != 'Nonbehavioral' run: | AZURE_CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_RESOURCE_GROUP=${{ env.RG_NAME }} make az-creds - CLUSTER_NAME=${{ env.CLUSTER_NAME }} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests + CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_ACR_NAME=${{ env.ACR_NAME}} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests - name: dump logs on failure uses: ./.github/actions/e2e/dump-logs if: failure() || cancelled() diff --git a/Makefile b/Makefile index d8204f6c4..3a93ed537 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ e2etests: ## Run the e2e suite against your local cluster # -count 1: prevents caching # -timeout: If a test binary runs longer than TEST_TIMEOUT, panic # -v: verbose output - cd test && CLUSTER_NAME=${CLUSTER_NAME} go test \ + cd test && CLUSTER_NAME=${CLUSTER_NAME} AZURE_ACR_NAME=${AZURE_ACR_NAME} go test \ -p 1 \ -count 1 \ -timeout ${TEST_TIMEOUT} \ diff --git a/Makefile-az.mk b/Makefile-az.mk index 5db0a0864..3019d9a9e 100755 --- a/Makefile-az.mk +++ b/Makefile-az.mk @@ -38,6 +38,15 @@ az-mkacr: az-mkrg ## Create test ACR --sku Basic --admin-enabled -o none az acr login --name $(AZURE_ACR_NAME) +az-acrimport: ## Imports an image to an acr registry + az acr import --name $(AZURE_ACR_NAME) --source "mcr.microsoft.com/oss/kubernetes/pause:3.6" --image "pause:3.6" + +az-cleanenv: az-rmnodeclaims-fin ## Deletes a few common karpenter testing resources(pods, nodepools, nodeclaims, aksnodeclasses) + kubectl delete pods -n default --all + kubectl delete nodeclaims --all + kubectl delete nodepools --all + kubectl delete aksnodeclasses --all + az-mkaks: az-mkacr ## Create test AKS cluster (with --vm-set-type AvailabilitySet for compatibility with standalone VMs) az aks create --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) --location $(AZURE_LOCATION) \ --enable-managed-identity --node-count 3 --generate-ssh-keys --vm-set-type AvailabilitySet -o none diff --git a/test/suites/acr/suite_test.go b/test/suites/acr/suite_test.go new file mode 100644 index 000000000..076b03ca0 --- /dev/null +++ b/test/suites/acr/suite_test.go @@ -0,0 +1,80 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package acr + +import ( + "fmt" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/test" +) + +var env *azure.Environment +var nodeClass *v1alpha2.AKSNodeClass +var nodePool *corev1beta1.NodePool +var pauseImage string + +func TestAcr(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = azure.NewEnvironment(t) + acrName := os.Getenv("AZURE_ACR_NAME") + Expect(acrName).NotTo(BeEmpty(), "AZURE_ACR_NAME must be set for the acr test suite") + pauseImage = fmt.Sprintf("%s.azurecr.io/pause:3.6", acrName) + }) + RunSpecs(t, "Acr") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultAKSNodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) + +var _ = Describe("Acr", func() { + Describe("Image Pull", func() { + It("should allow karpenter user pool nodes to pull images from the clusters attached acr", func() { + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 1, + PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + }, + Image: pauseImage, + }, + }) + + env.ExpectCreated(nodePool, nodeClass, deployment) + env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + }) + }) +})