From d29f7181a4f9165744b37cf1b5310987bf2bdb08 Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Sun, 28 Apr 2024 07:52:46 +0000 Subject: [PATCH] Use managed identity for aks pipeline * Update azure pipeline config * Update e2e to support managed identity --- .pipelines/ccm-image.yml | 8 +++-- .pipelines/cnm-image.yml | 8 +++-- .pipelines/daily-gc.yml | 8 ++--- .pipelines/run-autoscaling-e2e.yml | 3 +- .pipelines/run-autoscaling-multipool-e2e.yml | 3 +- .pipelines/run-basic-lb-e2e.yml | 3 +- .pipelines/run-e2e.yml | 6 ++-- .pipelines/run-vmas-e2e.yml | 3 +- .pipelines/scripts/daily-gc.sh | 2 +- .pipelines/scripts/run-e2e.sh | 27 +++++++++++----- tests/e2e/e2e_test.go | 7 +++++ tests/e2e/utils/azure_auth.go | 33 +++++++++++++++++--- 12 files changed, 79 insertions(+), 32 deletions(-) diff --git a/.pipelines/ccm-image.yml b/.pipelines/ccm-image.yml index ab26942ef2..a472147a30 100644 --- a/.pipelines/ccm-image.yml +++ b/.pipelines/ccm-image.yml @@ -6,7 +6,11 @@ steps: git show --stat echo $REGISTRY_PASSWORD | docker login $REGISTRY_URL -u $REGISTRY_USERNAME --password-stdin export IMAGE_REGISTRY=$REGISTRY_URL - export GOPATH="/home/vsts/go" + USER="cloudtest" + if [[ -n "${RELEASE_PIPELINE:-}" ]]; then + USER="vsts" + fi + export GOPATH="/home/${USER}/go" export PATH="${PATH}:${GOPATH}/bin" if [[ ! -d kubetest2-aks ]]; then @@ -19,7 +23,7 @@ steps: go install sigs.k8s.io/kubetest2@latest go mod tidy make deployer - sudo GOPATH="/home/vsts/go" make install + sudo GOPATH="/home/${USER}/go" make install popd export IMAGE_TAG="$(git describe --tags --match "v[0-9].*")" diff --git a/.pipelines/cnm-image.yml b/.pipelines/cnm-image.yml index 61d0698b54..23f26d790b 100644 --- a/.pipelines/cnm-image.yml +++ b/.pipelines/cnm-image.yml @@ -6,7 +6,11 @@ steps: git show --stat echo $REGISTRY_PASSWORD | docker login $REGISTRY_URL -u $REGISTRY_USERNAME --password-stdin export IMAGE_REGISTRY=$REGISTRY_URL - export GOPATH="/home/vsts/go" + USER="cloudtest" + if [[ -n "${RELEASE_PIPELINE:-}" ]]; then + USER="vsts" + fi + export GOPATH="/home/${USER}/go" export PATH="${PATH}:${GOPATH}/bin" if [[ ! -d kubetest2-aks ]]; then @@ -19,7 +23,7 @@ steps: go install sigs.k8s.io/kubetest2@latest go mod tidy make deployer - sudo GOPATH="/home/vsts/go" make install + sudo GOPATH="/home/${USER}/go" make install popd export IMAGE_TAG="$(git describe --tags --match "v[0-9].*")" diff --git a/.pipelines/daily-gc.yml b/.pipelines/daily-gc.yml index 126effb2da..c15c6900f3 100644 --- a/.pipelines/daily-gc.yml +++ b/.pipelines/daily-gc.yml @@ -10,6 +10,8 @@ trigger: none pr: none +pool: cloud-provider-azure-e2e-pool + variables: - template: var-e2e.yml @@ -23,13 +25,11 @@ stages: - bash: | export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID export AZURE_TENANT_ID=$AZ_TENANT_ID - export AZURE_CLIENT_ID=$SP_CLIENT_ID - export AZURE_CLIENT_SECRET=$SP_CLIENT_SECRET + export AZURE_MANAGED_IDENTITY_CLIENT_ID=$AZ_MANAGED_IDENTITY_CLIENT_ID .pipelines/scripts/daily-gc.sh displayName: gc aks and resource groups env: AZ_SUBSCRIPTION_ID: $(az.subscription_id) AZ_TENANT_ID: $(az.tenant_id) - SP_CLIENT_ID: $(sp.client_id) - SP_CLIENT_SECRET: $(sp.client_secret) + AZ_MANAGED_IDENTITY_PRINCIPAL_ID: $(az.mi_client_id) diff --git a/.pipelines/run-autoscaling-e2e.yml b/.pipelines/run-autoscaling-e2e.yml index 4204e59c19..d558e2b21c 100644 --- a/.pipelines/run-autoscaling-e2e.yml +++ b/.pipelines/run-autoscaling-e2e.yml @@ -9,8 +9,7 @@ trigger: none pr: none -pool: - vmImage: ubuntu-latest +pool: cloud-provider-azure-e2e-pool variables: - template: var-e2e.yml diff --git a/.pipelines/run-autoscaling-multipool-e2e.yml b/.pipelines/run-autoscaling-multipool-e2e.yml index 55c5718854..83dea08634 100644 --- a/.pipelines/run-autoscaling-multipool-e2e.yml +++ b/.pipelines/run-autoscaling-multipool-e2e.yml @@ -9,8 +9,7 @@ trigger: none pr: none -pool: - vmImage: ubuntu-latest +pool: cloud-provider-azure-e2e-pool variables: - template: var-e2e.yml diff --git a/.pipelines/run-basic-lb-e2e.yml b/.pipelines/run-basic-lb-e2e.yml index 13fbbee047..c9abbab901 100644 --- a/.pipelines/run-basic-lb-e2e.yml +++ b/.pipelines/run-basic-lb-e2e.yml @@ -12,8 +12,7 @@ trigger: pr: none -pool: - vmImage: ubuntu-latest +pool: cloud-provider-azure-e2e-pool variables: - template: var-e2e.yml diff --git a/.pipelines/run-e2e.yml b/.pipelines/run-e2e.yml index 7505e3e2b8..fdfc4f5c6c 100644 --- a/.pipelines/run-e2e.yml +++ b/.pipelines/run-e2e.yml @@ -8,9 +8,8 @@ steps: - bash: | export IMAGE_REGISTRY=$REGISTRY_URL export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID + export AZURE_MANAGED_IDENTITY_CLIENT_ID=$AZ_MANAGED_IDENTITY_CLIENT_ID export AZURE_TENANT_ID=$AZ_TENANT_ID - export AZURE_CLIENT_ID=$SP_CLIENT_ID - export AZURE_CLIENT_SECRET=$SP_CLIENT_SECRET export ARTIFACT_DIR=$BUILD_ARTIFACT_STAGING_DIRECTORY export KUSTO_INGESTION_URI=$KUSTO_INGESTION_URI @@ -24,9 +23,8 @@ steps: env: REGISTRY_URL: $(registry.url) AZ_SUBSCRIPTION_ID: $(az.subscription_id) + AZ_MANAGED_IDENTITY_PRINCIPAL_ID: $(az.mi_client_id) AZ_TENANT_ID: $(az.tenant_id) - SP_CLIENT_ID: $(sp.client_id) - SP_CLIENT_SECRET: $(sp.client_secret) BUILD_ARTIFACT_STAGING_DIRECTORY: $(Build.ArtifactStagingDirectory) BUILD_SOURCE_BRANCH_NAME: $(Build.SourceBranchName) KUSTO_INGESTION_URI: $(kusto.ingestion_uri) diff --git a/.pipelines/run-vmas-e2e.yml b/.pipelines/run-vmas-e2e.yml index 011df7132f..3cb8180eee 100644 --- a/.pipelines/run-vmas-e2e.yml +++ b/.pipelines/run-vmas-e2e.yml @@ -9,8 +9,7 @@ trigger: none pr: none -pool: - vmImage: ubuntu-latest +pool: cloud-provider-azure-e2e-pool variables: - template: var-e2e.yml diff --git a/.pipelines/scripts/daily-gc.sh b/.pipelines/scripts/daily-gc.sh index f1be65ce15..4847ec4e48 100755 --- a/.pipelines/scripts/daily-gc.sh +++ b/.pipelines/scripts/daily-gc.sh @@ -21,7 +21,7 @@ set -o pipefail CLUSTER_NAME="aks-cluster" CURRENT_DATE="$(date +%s)" -az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" +az login --identity --username "${AZURE_MANAGED_IDENTITY_CLIENT_ID:-}" az group list --tag usage=aks-cluster-e2e | jq -r '.[].name' | awk '{print $1}' | while read -r RESOURCE_GROUP; do RG_DATE="$(az group show --resource-group ${RESOURCE_GROUP} | jq -r '.tags.creation_date')" DATE_DIFF="$(expr ${CURRENT_DATE} - ${RG_DATE})" diff --git a/.pipelines/scripts/run-e2e.sh b/.pipelines/scripts/run-e2e.sh index 3793d01ea3..5dbf331941 100755 --- a/.pipelines/scripts/run-e2e.sh +++ b/.pipelines/scripts/run-e2e.sh @@ -19,14 +19,20 @@ set -o nounset set -o pipefail REPO_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/../..) -export GOPATH="/home/vsts/go" -export PATH="${PATH:-}:${GOPATH}/bin" -export AKS_CLUSTER_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID:-}/resourcegroups/${RESOURCE_GROUP:-}/providers/Microsoft.ContainerService/managedClusters/${CLUSTER_NAME:-}" - -if [[ -z "${RELEASE_PIPELINE:-}" ]]; then - az login --service-principal -u "${AZURE_CLIENT_ID:-}" -p "${AZURE_CLIENT_SECRET:-}" --tenant "${AZURE_TENANT_ID:-}" +USER="cloudtest" +if [[ -n "${RELEASE_PIPELINE:-}" ]]; then + # release pipeline uses sp + USER="vsts" +else + # aks pipeline uses managed identity + az login --identity --username "${AZURE_MANAGED_IDENTITY_CLIENT_ID:-}" + export E2E_MANAGED_IDENTITY_TYPE="userassigned" fi +export GOPATH="/home/${USER}/go" +export PATH="${PATH}:${GOPATH}/bin" +export AKS_CLUSTER_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID:-}/resourcegroups/${RESOURCE_GROUP:-}/providers/Microsoft.ContainerService/managedClusters/${CLUSTER_NAME:-}" + get_random_location() { local LOCATIONS=("eastus") echo "${LOCATIONS[${RANDOM} % ${#LOCATIONS[@]}]}" @@ -125,7 +131,7 @@ if [[ "${SKIP_BUILD_KUBETEST2_AKS:-}" != "true" ]]; then if [[ -n "${RELEASE_PIPELINE:-}" ]]; then make install else - sudo GOPATH="/home/vsts/go" make install + sudo GOPATH="/home/${USER}/go" make install fi rm /tmp/cloud-provider-azure -rf popd @@ -181,6 +187,13 @@ if [[ "${CLUSTER_TYPE:-}" =~ "autoscaling" ]]; then fi fi +# Do role assignment for the managed identity +if [[ -z "${RELEASE_PIPELINE:-}" ]]; then + MC_RESOURCE_ID="/subscriptions/${AZURE_SUBSCRIPTION_ID:-}/resourceGroups/MC_${RESOURCE_GROUP:-}_${CLUSTER_NAME:-}_eastus" + AGENTPOOL_MI_PRINCIPAL_ID="$(az identity show -n ${CLUSTER_NAME:-}-agentpool -g MC_${RESOURCE_GROUP:-}_${CLUSTER_NAME:-}_eastus --subscription ${AZURE_SUBSCRIPTION_ID:-} | jq -r '.principalId')" + az role assignment create --assignee "${AGENTPOOL_MI_PRINCIPAL_ID}" --role "AcrPull" --scope "${MC_RESOURCE_ID}" --subscription "${AZURE_SUBSCRIPTION_ID:-}" +fi + if [[ "${SKIP_E2E:-}" != "true" ]]; then echo "Running e2e" export E2E_ON_AKS_CLUSTER=true diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index f9e54d77ef..e9b2b4b117 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -100,6 +100,13 @@ func TestAzureTest(t *testing.T) { if !strings.EqualFold(os.Getenv(utils.IngestTestResult), utils.TrueValue) { return } + + // TODO: Use managed identity for Kusto ingestion + skipKustoIngestion := true + if skipKustoIngestion { + return + } + klog.Infof("Ingesting test result to kusto") if err := utils.KustoIngest(passed, suiteConfig.LabelFilter, os.Getenv(utils.AKSClusterType), reporterConfig.JUnitReport); err != nil { klog.Error(err) diff --git a/tests/e2e/utils/azure_auth.go b/tests/e2e/utils/azure_auth.go index 0cc4657175..2fef7620f5 100644 --- a/tests/e2e/utils/azure_auth.go +++ b/tests/e2e/utils/azure_auth.go @@ -17,6 +17,7 @@ limitations under the License. package utils import ( + "fmt" "os" "sigs.k8s.io/cloud-provider-azure/pkg/azclient" @@ -31,6 +32,10 @@ const ( ClusterLocationEnv = "AZURE_LOCATION" ClusterEnvironment = "AZURE_ENVIRONMENT" LoadBalancerSkuEnv = "AZURE_LOADBALANCER_SKU" + managedIdentityClientID = "AZURE_MANAGED_IDENTITY_CLIENT_ID" + managedIdentityType = "E2E_MANAGED_IDENTITY_TYPE" + + userAssignedManagedIdentity = "userassigned" ) // azureAuthConfigFromTestProfile obtains azure config from Environment @@ -39,10 +44,30 @@ func azureAuthConfigFromTestProfile() (*azclient.AzureAuthConfig, *azclient.ARMC if len(envStr) == 0 { envStr = "AZUREPUBLICCLOUD" } - return &azclient.AzureAuthConfig{ - AADClientID: os.Getenv(ServicePrincipleIDEnv), - AADClientSecret: os.Getenv(ServicePrincipleSecretEnv), - }, &azclient.ARMClientConfig{ + + var azureAuthConfig azclient.AzureAuthConfig + servicePrincipleIDEnv := os.Getenv(ServicePrincipleIDEnv) + servicePrincipleSecretEnv := os.Getenv(ServicePrincipleSecretEnv) + managedIdentityTypeEnv := os.Getenv(managedIdentityType) + managedIdentityClientIDEnv := os.Getenv(managedIdentityClientID) + if servicePrincipleIDEnv != "" && servicePrincipleSecretEnv != "" { + azureAuthConfig = azclient.AzureAuthConfig{ + AADClientID: servicePrincipleIDEnv, + AADClientSecret: servicePrincipleSecretEnv, + } + } else if managedIdentityTypeEnv != "" { + azureAuthConfig = azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + } + if managedIdentityTypeEnv == userAssignedManagedIdentity { + azureAuthConfig.UserAssignedIdentityID = managedIdentityClientIDEnv + + } + } else { + return nil, nil, nil, fmt.Errorf("failed to get Azure auth config from environment") + } + + return &azureAuthConfig, &azclient.ARMClientConfig{ Cloud: envStr, TenantID: os.Getenv(TenantIDEnv), }, &azclient.ClientFactoryConfig{