From 73d9b1fe76d0dd5c23678cc9b5db180465f1b6d6 Mon Sep 17 00:00:00 2001 From: Chad Kittel Date: Fri, 16 Sep 2022 13:26:45 -0500 Subject: [PATCH] Replace starter github workflow with dedicated repo reference (#348) * commit to see how intro table looks * add newlines instead * remove github workflow and add two more links to automation repo * Apply suggestions from code review Co-authored-by: Rick Hallihan <1796255+hallihan@users.noreply.github.com> * Update 12-cleanup.md Co-authored-by: Rick Hallihan <1796255+hallihan@users.noreply.github.com> Co-authored-by: Rick Hallihan <1796255+hallihan@users.noreply.github.com> --- .gitignore | 1 - 01-prerequisites.md | 3 + 06-aks-cluster.md | 94 ------------------ 12-cleanup.md | 8 ++ README.md | 2 +- github-workflow/README.md | 26 ----- github-workflow/aks-deploy.properties.json | 12 --- github-workflow/aks-deploy.yaml | 110 --------------------- 8 files changed, 12 insertions(+), 244 deletions(-) delete mode 100644 github-workflow/README.md delete mode 100644 github-workflow/aks-deploy.properties.json delete mode 100644 github-workflow/aks-deploy.yaml diff --git a/.gitignore b/.gitignore index 9f489f86..8c3833c8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,4 @@ *.key *.pfx *.pem -sp.json aks_baseline.env diff --git a/01-prerequisites.md b/01-prerequisites.md index af3e95c9..c68b29e1 100644 --- a/01-prerequisites.md +++ b/01-prerequisites.md @@ -2,6 +2,9 @@ This is the starting point for the instructions on deploying the [AKS Baseline reference implementation](./README.md). There is required access and tooling you'll need in order to accomplish this. Follow the instructions below and on the subsequent pages so that you can get your environment ready to proceed with the AKS cluster creation. +| :clock10: | These steps are intentionally verbose, intermixed with context, narrative, and guidance. The deployments are all conducted via [Bicep templates](https://learn.microsoft.com/azure/azure-resource-manager/bicep/overview), but they are executed manually via `az cli` commands. We strongly encourage you to dedicate time to walk through these instructions, with a focus on learning. We do not provide any "one click" method to complete all deployments.

Once you understand the components involved and have identified the shared responsibilities between your team and your greater organization, you are encouraged to build suitable, repeatable deployment processes around your final infrastructure and cluster bootstrapping. The [AKS baseline automation guidance](https://github.com/Azure/aks-baseline-automation#aks-baseline-automation) is a great place to learn how to build your own automation pipelines. That guidance is based on the same architecture foundations presented here in the AKS baseline, and illustrates GitHub Actions based deployments for all components, including workloads. | +|-----------|:--------------------------| + ## Steps 1. An Azure subscription. diff --git a/06-aks-cluster.md b/06-aks-cluster.md index 20f4f432..2c17c031 100644 --- a/06-aks-cluster.md +++ b/06-aks-cluster.md @@ -19,8 +19,6 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi 1. Deploy the cluster ARM template. :exclamation: By default, this deployment will allow unrestricted access to your cluster's API Server. You can limit access to the API Server to a set of well-known IP addresses (i.,e. a jump box subnet (connected to by Azure Bastion), build agents, or any other networks you'll administer the cluster from) by setting the `clusterAuthorizedIPRanges` parameter in all deployment options. This setting will also impact traffic originating from within the cluster trying to use the API server, so you will also need to include _all_ of the public IPs used by your egress Azure Firewall. For more information, see [Secure access to the API server using authorized IP address ranges](https://learn.microsoft.com/azure/aks/api-server-authorized-ip-ranges#create-an-aks-cluster-with-api-server-authorized-ip-ranges-enabled). - **Option 1 - Deploy from the command line** - ```bash # [This takes about 18 minutes.] az deployment group create -g rg-bu0001a0008 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE} a0008NamespaceReaderAadGroupObjectId=${AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC_AKS_BASELINE} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE} domainName=${DOMAIN_NAME_AKS_BASELINE} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME} location=eastus2 @@ -28,98 +26,6 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi > Alteratively, you could have updated the [`azuredeploy.parameters.prod.json`](./azuredeploy.parameters.prod.json) file and deployed as above, using `-p "@azuredeploy.parameters.prod.json"` instead of providing the individual key-value pairs. - **Option 2 - Automated deploy using GitHub Actions (fork is required)** - - 1. Create the Azure Credentials for the GitHub CD workflow. - - ```bash - # Create an Azure Service Principal - # - # This command will generate a sp.json file in the format expected by GitHub Actions for Azure. - # This is accomplished by using the deprecated --sdk-auth flag. For alternatives, including using - # Federated Identity, see https://github.com/Azure/login#configure-deployment-credentials. - az ad sp create-for-rbac --name "github-workflow-aks-cluster" --sdk-auth --skip-assignment > sp.json - export APP_ID=$(grep -oP '(?<="clientId": ").*?[^\\](?=",)' sp.json) - echo APP_ID: $APP_ID - - # Wait for propagation - until az ad sp show --id ${APP_ID} &> /dev/null ; do echo "Waiting for Azure AD propagation" && sleep 5; done - - # Assign built-in Contributor RBAC role for creating resource groups and performing deployments at subscription level - az role assignment create --assignee $APP_ID --role 'Contributor' - - # Assign built-in User Access Administrator RBAC role since granting RBAC access to other resources during the cluster creation will be required at subscription level (e.g. AKS-managed Internal Load Balancer, ACR, Managed Identities, etc.) - az role assignment create --assignee $APP_ID --role 'User Access Administrator' - ``` - - 1. Create `AZURE_CREDENTIALS` secret in your GitHub repository. For more - information, please take a look at [Creating encrypted secrets for a repository](https://docs.github.com/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets-for-a-repository). - - > :bulb: Use the content from the `sp.json` file. - - ```bash - cat sp.json - ``` - - 1. Create `APP_GATEWAY_LISTENER_CERTIFICATE_BASE64` secret in your GitHub repository. For more - information, please take a look at [Creating encrypted secrets for a repository](https://docs.github.com/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets-for-a-repository). - - > :bulb: - > - > * Use the env var value of `APP_GATEWAY_LISTENER_CERTIFICATE` - > * Ideally fetching this secret from a platform-managed secret store such as [Azure KeyVault](https://github.com/marketplace/actions/azure-key-vault-get-secrets) - - ```bash - echo $APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE - ``` - - 1. Create `AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64` secret in your GitHub repository. For more information, please take a look at [Creating encrypted secrets for a repository](https://docs.github.com/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets-for-a-repository). - - > :bulb: - > - > * Use the env var value of `AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64` - > * Ideally fetching this secret from a platform-managed secret store such as [Azure Key Vault](https://github.com/marketplace/actions/azure-key-vault-get-secrets) - - ```bash - echo $AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE - ``` - - 1. Copy the GitHub workflow file into the expected directory and update the placeholders in it. - - ```bash - mkdir -p .github/workflows - cat github-workflow/aks-deploy.yaml | \ - sed "s##eastus2#g" | \ - sed "s##rg-bu0001a0008#g" | \ - sed "s##centralus#g" | \ - sed "s##${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE}#g" | \ - sed "s##${TENANTID_K8SRBAC_AKS_BASELINE}#g" | \ - sed "s##${AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE}#g" | \ - sed "s##${AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE}#g" | \ - sed "s##${DOMAIN_NAME_AKS_BASELINE}#g" | \ - sed "s##${GITOPS_REPOURL}#g" \ - > .github/workflows/aks-deploy.yaml - ``` - - 1. Push the changes to your forked repo. - - > :book: The DevOps team wants to automate their infrastructure deployments. In this case, they decided to use GitHub Actions. They are going to create a workflow for every AKS cluster instance they have to take care of. - - ```bash - git add .github/workflows/aks-deploy.yaml && git commit -m "setup GitHub CD workflow" - git push origin HEAD:kick-off-workflow - ``` - - > :bulb: You might want to convert this GitHub workflow into a template since your organization or team might need to handle multiple AKS clusters. For more information, please take a look at [Sharing Workflow Templates within your organization](https://docs.github.com/actions/configuring-and-managing-workflows/sharing-workflow-templates-within-your-organization). - - 1. Navigate to your GitHub forked repository and open a PR against `main` using the recently pushed changes to the remote branch `kick-off-workflow`. - - > :book: The DevOps team configured the GitHub Workflow to preview the changes that will happen when a PR is opened. This will allow them to evaluate the changes before they get deployed. After the PR reviewers see how resources will change if the AKS cluster ARM template gets deployed, it is possible to merge or discard the pull request. If the decision is made to merge, it will trigger a push event that will kick off the actual deployment process. - - 1. Once the GitHub Workflow validation finished successfully, please proceed by merging this PR into `main`. - - > :book: The DevOps team monitors this Workflow execution instance. In this instance it will impact a critical piece of infrastructure as well as the management. This flow works for both new or an existing AKS cluster. - ## Container registry note :warning: To aid in ease of deployment of this cluster and your experimentation with workloads, Azure Policy and Azure Firewall are currently configured to allow your cluster to pull images from _public container registries_ such as Docker Hub. For a production system, you'll want to update Azure Policy parameter named `allowedContainerImagesRegex` in your `cluster-stamp.bicep` file to only list those container registries that you are willing to take a dependency on and what namespaces those policies apply to, and make Azure Firewall allowances for the same. This will protect your cluster from unapproved registries being used, which may prevent issues while trying to pull images from a registry which doesn't provide SLA guarantees for your deployment. diff --git a/12-cleanup.md b/12-cleanup.md index 60cf2298..3b0a2cdd 100644 --- a/12-cleanup.md +++ b/12-cleanup.md @@ -28,6 +28,14 @@ After you are done exploring your deployed [AKS Baseline cluster](./), you'll wa 1. [Remove the Azure Policy assignments](https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyMenuBlade/Compliance) scoped to the cluster's resource group. To identify those created by this implementation, look for ones that are prefixed with `[your-cluster-name] `. +## Automation + +Before you can automate a process, it's important to experience the process in a bit more raw form as was presented here. That experience allows you to understand the various steps, inner- & cross-team dependencies, and failure points along the way. However, the steps provided in this walkthrough are not specifically designed with automation in mind. It does present a perspective on some common seperation of duties often encountered in organizations, but that might not align with your organization. + +Now that you understand the components involved and have identified the shared responsibilities between your team and your greater organization, you are encouraged to build repeatable deployment processes around your final infrastructure and cluster bootstrapping. Please refer to the [AKS baseline automation guidance](https://github.com/Azure/aks-baseline-automation#aks-baseline-automation) to learn how GitHub Actions combined with Infrastructure as Code can be used to facilitate this automation. That guidance is based on the same architecture foundations you've walked through here. + +> Note: The [AKS baseline automation guidance](https://github.com/Azure/aks-baseline-automation#aks-baseline-automation) implementation strives to stay in sync with this repo, but may slightly deviate in various decisions made, may introduce new features, or not yet have a feature that is used in this repo. The are functionally aligned by design, but not necessarily identical. Use that repo to explore the automation potential, while this repo is used for the core architectural guidance. + ### Next step :arrow_forward: [Review additional information in the main README](./README.md#broom-clean-up-resources) diff --git a/README.md b/README.md index e4ac7ab8..19dea2e4 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ This is the heart of the guidance in this reference implementation; paired with - [ ] [Deploy the AKS cluster and supporting services](./06-aks-cluster.md) - [ ] [Validate cluster bootsrapping](./07-bootstrap-validation.md) -We perform the prior steps manually here for you to understand the involved components, but we advocate for an automated DevOps process. Therefore, incorporate the prior steps into your CI/CD pipeline, as you would any infrastructure as code (IaC). We have included [a starter GitHub workflow](./github-workflow/aks-deploy.yaml) that demonstrates this. +We perform the prior steps manually here for you to understand the involved components, but we advocate for an automated DevOps process. Therefore, incorporate the prior steps into your CI/CD pipeline, as you would any infrastructure as code (IaC). See the dedicated [AKS baseline automation guidance](https://github.com/Azure/aks-baseline-automation#aks-baseline-automation) for additional details. ### 4. Deploy your workload diff --git a/github-workflow/README.md b/github-workflow/README.md deleted file mode 100644 index 6e950bf0..00000000 --- a/github-workflow/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# GitHub Actions Workflow - -> Note: This is part of the Azure Kubernetes Service (AKS) Baseline cluster reference implementation. For more information check out the [readme file in the root](../README.md). - -This cluster, as with any workload, should be managed via an automated deployment pipeline. In this reference implementation we provide a "getting started" GitHub Action workflow file that you can reference to build your own. - -## Steps - -This workflow file deploys the cluster into an already-existing virtual network and AAD configuration as set up by the steps in the [main README.md file](../README.md). - -## Secrets - -Secrets should not be stored in this file, but instead should be stored as part of the secret store of GitHub Actions. - -## Workload - -The workload is NOT part of this deployment. This is a deployment of the infrastructure only. Separation of infrastructure and workload is recommended as it allows you to have distinct lifecycle and operational concerns. - -## Next Steps - -Review the yaml file to see the types of steps you'd need to perform. Also consider your workflow as a mechanism to deploy to another region in case of a regional failure, the pipeline should be built in such a way that with parameter/input alterations, you can deploy a new cluster in a new region. - -## See also - -* [GitHub Actions](https://help.github.com/actions) -* [GitHub Actions with Azure Kubernetes Service](https://learn.microsoft.com/azure/aks/kubernetes-action) \ No newline at end of file diff --git a/github-workflow/aks-deploy.properties.json b/github-workflow/aks-deploy.properties.json deleted file mode 100644 index 35757c29..00000000 --- a/github-workflow/aks-deploy.properties.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "Deploy AKS Baseline cluster stamp", - "description": "This workflow will deploy our cluster stamp in your own vnet", - "creator": "Microsoft Patterns & Practices", - "iconName": "azure", - "categories": ["AKS"], - "filePatterns": [ - "cluster-stamp.json$", - "cluster-manifests/cluster-baseline-settings/flux.yaml$", - "cluster-manifests/*" - ] -} diff --git a/github-workflow/aks-deploy.yaml b/github-workflow/aks-deploy.yaml deleted file mode 100644 index bfbdfd92..00000000 --- a/github-workflow/aks-deploy.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# This workflow will deploy our bootstrapped cluster stamp, without the workload. -# -# Follow the next steps to use this workflow: -# -# 1. Your repository has the following structure. -# . -# ├── .github -# │   ├── workflows -# │   │   └── aks-deploy.yaml -# ├── cluster-manifests -# │   ├── cluster-baseline-settings/* -# └── cluster-stamp.bicep -# -# 2. Ensure you have followed the prior sections before deploying this AKS cluster. This way, you will be capable of setting: -# - the secrets values as detailed in the next step. -# - the environment variables from the env dictionary at the worklflow level. -# -# 3. Create the following secrets in your GitHub repository: -# - AZURE_CREDENTIALS The Azure Service Principal that will deploy the AKS cluster in your Azure subscription. For more information please take a look at https://github.com/Azure/login#configure-deployment-credentials -# - APP_GATEWAY_LISTENER_CERTIFICATE_BASE64 The certificate data for app gateway TLS termination. It is base64. Ideally fetch this secret from a platform-managed secret store such as Azure KeyVault: https://github.com/marketplace/actions/azure-key-vault-get-secrets -# - AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64 The Base64 encoded AKS Ingress Controller public certificate (as .crt or .cer) to be stored in Azure Key Vault as secret and referenced by Azure Application Gateway as a trusted root certificate. - -name: Deploy AKS Baseline cluster stamp - -on: - push: - paths: - - 'cluster-stamp.bicep' - - '.github/workflows/aks-deploy.yaml' - branches: [ main ] - pull_request: - paths: - - 'cluster-stamp.bicep' - - '.github/workflows/aks-deploy.yaml' - branches: [ main ] - -env: - RESOURCE_GROUP_LOCATION: '' # The location where the resource group is going to be created - RESOURCE_GROUP: '' # The name for the AKS cluster resource group - AKS_LOCATION: '' # The location where the AKS cluster is going to be deployed - GEO_REDUNDANCY_LOCATION: '' # The location for Azure resources that support native geo-redunancy. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/en-us/azure/best-practices-availability-paired-regions. This region does not need to support availability zones. - TARGET_VNET_RESOURCE_ID: '' # The regional network spoke VNet Resource ID that the cluster will be joined to - K8S_RBAC_AAD_PROFILE_TENANTID: '' # The tenant to integrate AKS-managed Azure AD - K8S_RBAC_AAD_PROFILE_ADMIN_GROUP_OBJECTID: '' # The Azure AD group object ID that has admin access to the AKS cluster - K8S_RBAC_AAD_A0008_READER_GROUP_OBJECTID: '' # The Azure AD group object ID that has readonly access to the a0008 namespace in the AKS cluster - CLUSTER_AUTHORIZED_IP_RANGES: '[]' # By default, this deployment will allow unrestricted access to your cluster's API Server. You should limit access to the API Server to a set of well-known IP addresses (i.,e. your hub firewall IP, bastion subnet, build agents, or any other networks you'll administer the cluster from), and can do so by adding a CLUSTER_AUTHORIZED_IP_RANGES="['managementRange1', 'managementRange2', 'AzureFirewallIP/32']"" parameter. - DOMAIN_NAME: '' # The domain name to use for App Gateway and AKS ingress. - BOOTSTRAPPING_REPO_HTTPS_URL: '' # The git https repo that will be used for bootstrapping your cluster -jobs: - deploy: - name: Deploy AKS cluster and Flux - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - # Login into your Azure Subscription using your Azure credentials - make sure the credentials has write permissions for the specific resource group/subscription. The credentials should be stored in GitHub Secrets - (Go to Settings Tab ->Secrets) - - name: Azure Login - uses: Azure/login@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - # Deploy the cluster into your environment, assuming all prerequisites are up and running. - - name: Azure CLI - Deploy AKS cluster - id: aks-cluster - uses: Azure/cli@v1.0.0 - with: - inlineScript: | - az group create --name ${{ env.RESOURCE_GROUP }} --location ${{ env.RESOURCE_GROUP_LOCATION }} - az deployment group $([[ ${{ github.event_name }} = pull_request ]] && echo what-if --no-pretty-print || echo create) \ - --resource-group ${{ env.RESOURCE_GROUP }} \ - --template-file "cluster-stamp.bicep" \ - --name "cluster-stamp" \ - --parameters \ - location=${{ env.AKS_LOCATION }} \ - geoRedundancyLocation=${{ env.GEO_REDUNDANCY_LOCATION }} \ - targetVnetResourceId=${{ env.TARGET_VNET_RESOURCE_ID }} \ - k8sControlPlaneAuthorizationTenantId=${{ env.K8S_RBAC_AAD_PROFILE_TENANTID }} \ - clusterAdminAadGroupObjectId=${{ env.K8S_RBAC_AAD_PROFILE_ADMIN_GROUP_OBJECTID }} \ - a0008NamespaceReaderAadGroupObjectId=${{ env.K8S_RBAC_AAD_A0008_READER_GROUP_OBJECTID }} \ - clusterAuthorizedIPRanges=${{ env.CLUSTER_AUTHORIZED_IP_RANGES}} \ - appGatewayListenerCertificate=${{ secrets.APP_GATEWAY_LISTENER_CERTIFICATE_BASE64 }} \ - aksIngressControllerCertificate=${{ secrets.AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64 }} \ - domainName=${{ env.DOMAIN_NAME }} \ - gitOpsBootstrappingRepoHttpsUrl=${{ BOOTSTRAPPING_REPO_HTTPS_URL }} - - echo "::set-output name=name::$(az deployment group show --resource-group ${{ env.RESOURCE_GROUP }} -n cluster-stamp --query properties.outputs.aksClusterName.value -o tsv)" - azcliversion: 2.29.2 - - # If you needed to do any post-deployment bootstrapping that was not already set up you can follow - # a pattern like the following. They don't do anything meaningful, but are left here as a quickstart - # guide for your own post-deployment, pipeline-based configuration - - name: Set the AKS cluster context - uses: Azure/aks-set-context@v1 - if: github.event_name == 'push' - with: - creds: '${{ secrets.AZURE_CREDENTIALS }}' - cluster-name: ${{ steps.aks-cluster.outputs.name }} - resource-group: ${{ env.RESOURCE_GROUP }} - - # Apply a manifest file from this repo. Technically this manifest file does NOT need to be - # applied. It is a namespace manifest which was already deployed as part of the bootstrapping - # process. - - name: Create the cluster-baseline-settings namespace - uses: Azure/k8s-deploy@v1 - if: github.event_name == 'push' - with: - namespace: 'cluster-baseline-settings' - manifests: | - cluster-manifests/cluster-baseline-settings/ns-cluster-baseline-settings.yaml