From 2c126ec5c5e5eb43dd878bb3bc29999a70a1bb42 Mon Sep 17 00:00:00 2001 From: Matthew Frahry Date: Fri, 13 Dec 2019 15:11:56 -0800 Subject: [PATCH 1/5] Start of kubernetes MSI work --- azurerm/resource_arm_kubernetes_cluster.go | 72 +++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/azurerm/resource_arm_kubernetes_cluster.go b/azurerm/resource_arm_kubernetes_cluster.go index 556381039e03..2a9b17544cce 100644 --- a/azurerm/resource_arm_kubernetes_cluster.go +++ b/azurerm/resource_arm_kubernetes_cluster.go @@ -560,6 +560,32 @@ func resourceArmKubernetesCluster() *schema.Resource { Computed: true, Sensitive: true, }, + + "managed_cluster_identity": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "type": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{ + string(containerservice.None), + string(containerservice.SystemAssigned), + }, false), + }, + "principal_id": { + Type: schema.TypeString, + Computed: true, + }, + "tenant_id": { + Type: schema.TypeString, + Computed: true, + }, + }, + }, + }, }, } } @@ -640,6 +666,9 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{} enablePodSecurityPolicy := d.Get("enable_pod_security_policy").(bool) + managedClusterIdentityRaw := d.Get("managed_cluster_identity").([]interface{}) + managedClusterIdentity := expandKubernetesClusterManagedClusterIdentity(managedClusterIdentityRaw) + parameters := containerservice.ManagedCluster{ Name: &name, Location: &location, @@ -658,7 +687,8 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{} NodeResourceGroup: utils.String(nodeResourceGroup), EnablePodSecurityPolicy: utils.Bool(enablePodSecurityPolicy), }, - Tags: tags.Expand(t), + Identity: managedClusterIdentity, + Tags: tags.Expand(t), } future, err := client.CreateOrUpdate(ctx, resGroup, name, parameters) @@ -792,6 +822,13 @@ func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{} existing.ManagedClusterProperties.WindowsProfile = windowsProfile } + if d.HasChange("managed_cluster_identity") { + updateCluster = true + managedClusterIdentityRaw := d.Get("managed_cluster_identity").([]interface{}) + managedClusterIdentity := expandKubernetesClusterManagedClusterIdentity(managedClusterIdentityRaw) + existing.Identity = managedClusterIdentity + } + if updateCluster { log.Printf("[DEBUG] Updating the Kubernetes Cluster %q (Resource Group %q)..", name, resourceGroup) future, err := clusterClient.CreateOrUpdate(ctx, resourceGroup, name, existing) @@ -974,6 +1011,12 @@ func resourceArmKubernetesClusterRead(d *schema.ResourceData, meta interface{}) } } + if identity := resp.Identity; identity != nil { + if err := d.Set("managed_cluster_identity", flattenKubernetesClusterManagedClusterIdentity(*identity)); err != nil { + return fmt.Errorf("Error setting `managed_cluster_identity`: %+v", err) + } + } + kubeConfigRaw, kubeConfig := flattenKubernetesClusterAccessProfile(profile) d.Set("kube_config_raw", kubeConfigRaw) if err := d.Set("kube_config", kubeConfig); err != nil { @@ -1403,6 +1446,20 @@ func expandKubernetesClusterRoleBasedAccessControl(input []interface{}, provider return rbacEnabled, aad } +func expandKubernetesClusterManagedClusterIdentity(input []interface{}) *containerservice.ManagedClusterIdentity { + if len(input) == 0 || input[0] == nil { + return nil + } + + val := input[0].(map[string]interface{}) + + managedClusterIdentity := &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(val["type"].(string)), + } + + return managedClusterIdentity +} + func flattenKubernetesClusterRoleBasedAccessControl(input *containerservice.ManagedClusterProperties, d *schema.ResourceData) []interface{} { rbacEnabled := false if input.EnableRBAC != nil { @@ -1529,3 +1586,16 @@ func flattenKubernetesClusterKubeConfigAAD(config kubernetes.KubeConfigAAD) []in }, } } + +func flattenKubernetesClusterManagedClusterIdentity(input containerservice.ManagedClusterIdentity) []interface{} { + identity := make(map[string]interface{}, 0) + if input.PrincipalID != nil { + identity["principal_id"] = *input.PrincipalID + } + if input.TenantID != nil { + identity["tenant_id"] = *input.TenantID + } + identity["type"] = string(input.Type) + + return []interface{}{identity} +} From b9cbbfc47649d5b19d060f529d47ee4bbaa3cd8d Mon Sep 17 00:00:00 2001 From: Matthew Frahry Date: Fri, 13 Dec 2019 17:00:22 -0800 Subject: [PATCH 2/5] Add test/docs for managed cluster identity --- azurerm/resource_arm_kubernetes_cluster.go | 14 +++-- ...ource_arm_kubernetes_cluster_other_test.go | 60 +++++++++++++++++++ .../resource_arm_kubernetes_cluster_test.go | 19 +++--- .../docs/r/kubernetes_cluster.html.markdown | 18 +++++- 4 files changed, 96 insertions(+), 15 deletions(-) diff --git a/azurerm/resource_arm_kubernetes_cluster.go b/azurerm/resource_arm_kubernetes_cluster.go index 2a9b17544cce..6157245cf332 100644 --- a/azurerm/resource_arm_kubernetes_cluster.go +++ b/azurerm/resource_arm_kubernetes_cluster.go @@ -565,11 +565,13 @@ func resourceArmKubernetesCluster() *schema.Resource { Type: schema.TypeList, Optional: true, MaxItems: 1, + ForceNew: true, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "type": { Type: schema.TypeString, Required: true, + ForceNew: true, ValidateFunc: validation.StringInSlice([]string{ string(containerservice.None), string(containerservice.SystemAssigned), @@ -1011,10 +1013,8 @@ func resourceArmKubernetesClusterRead(d *schema.ResourceData, meta interface{}) } } - if identity := resp.Identity; identity != nil { - if err := d.Set("managed_cluster_identity", flattenKubernetesClusterManagedClusterIdentity(*identity)); err != nil { - return fmt.Errorf("Error setting `managed_cluster_identity`: %+v", err) - } + if err := d.Set("managed_cluster_identity", flattenKubernetesClusterManagedClusterIdentity(resp.Identity)); err != nil { + return fmt.Errorf("Error setting `managed_cluster_identity`: %+v", err) } kubeConfigRaw, kubeConfig := flattenKubernetesClusterAccessProfile(profile) @@ -1587,7 +1587,11 @@ func flattenKubernetesClusterKubeConfigAAD(config kubernetes.KubeConfigAAD) []in } } -func flattenKubernetesClusterManagedClusterIdentity(input containerservice.ManagedClusterIdentity) []interface{} { +func flattenKubernetesClusterManagedClusterIdentity(input *containerservice.ManagedClusterIdentity) []interface{} { + if input == nil { + return []interface{}{} + } + identity := make(map[string]interface{}, 0) if input.PrincipalID != nil { identity["principal_id"] = *input.PrincipalID diff --git a/azurerm/resource_arm_kubernetes_cluster_other_test.go b/azurerm/resource_arm_kubernetes_cluster_other_test.go index 940f8f8f1985..c0b31f190063 100644 --- a/azurerm/resource_arm_kubernetes_cluster_other_test.go +++ b/azurerm/resource_arm_kubernetes_cluster_other_test.go @@ -323,6 +323,34 @@ func testAccAzureRMKubernetesCluster_windowsProfile(t *testing.T) { }) } +func testAccAzureRMKubernetesCluster_managedClusterIdentiy(t *testing.T) { + resourceName := "azurerm_kubernetes_cluster.test" + ri := tf.AccRandTimeInt() + clientId := os.Getenv("ARM_CLIENT_ID") + clientSecret := os.Getenv("ARM_CLIENT_SECRET") + location := testLocation() + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testCheckAzureRMKubernetesClusterDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAzureRMKubernetesCluster_managedClusterIdentityConfig(ri, clientId, clientSecret, location), + Check: resource.ComposeTestCheckFunc( + testCheckAzureRMKubernetesClusterExists(resourceName), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"service_principal.0.client_secret"}, + }, + }, + }) +} + func testAccAzureRMKubernetesCluster_basicAvailabilitySetConfig(rInt int, clientId string, clientSecret string, location string) string { return fmt.Sprintf(` resource "azurerm_resource_group" "test" { @@ -642,3 +670,35 @@ resource "azurerm_kubernetes_cluster" "test" { } `, rInt, location, rInt, rInt, rInt, clientId, clientSecret) } + +func testAccAzureRMKubernetesCluster_managedClusterIdentityConfig(rInt int, clientId string, clientSecret string, location string) string { + return fmt.Sprintf(` +resource "azurerm_resource_group" "test" { + name = "acctestRG-%d" + location = "%s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + + default_node_pool { + name = "default" + node_count = 1 + type = "AvailabilitySet" + vm_size = "Standard_DS2_v2" + } + + service_principal { + client_id = "%s" + client_secret = "%s" + } + + managed_cluster_identity { + type = "SystemAssigned" + } +} +`, rInt, location, rInt, rInt, clientId, clientSecret) +} diff --git a/azurerm/resource_arm_kubernetes_cluster_test.go b/azurerm/resource_arm_kubernetes_cluster_test.go index d64e156e2b58..65902807900f 100644 --- a/azurerm/resource_arm_kubernetes_cluster_test.go +++ b/azurerm/resource_arm_kubernetes_cluster_test.go @@ -70,15 +70,16 @@ func TestAccAzureRMKubernetes_all(t *testing.T) { "windowsAndLinux": testAccAzureRMKubernetesClusterNodePool_windowsAndLinux, }, "other": { - "basicAvailabilitySet": testAccAzureRMKubernetesCluster_basicAvailabilitySet, - "basicVMSS": testAccAzureRMKubernetesCluster_basicVMSS, - "requiresImport": testAccAzureRMKubernetesCluster_requiresImport, - "linuxProfile": testAccAzureRMKubernetesCluster_linuxProfile, - "nodeTaints": testAccAzureRMKubernetesCluster_nodeTaints, - "nodeResourceGroup": testAccAzureRMKubernetesCluster_nodeResourceGroup, - "upgradeConfig": testAccAzureRMKubernetesCluster_upgrade, - "tags": testAccAzureRMKubernetesCluster_tags, - "windowsProfile": testAccAzureRMKubernetesCluster_windowsProfile, + "basicAvailabilitySet": testAccAzureRMKubernetesCluster_basicAvailabilitySet, + "basicVMSS": testAccAzureRMKubernetesCluster_basicVMSS, + "requiresImport": testAccAzureRMKubernetesCluster_requiresImport, + "linuxProfile": testAccAzureRMKubernetesCluster_linuxProfile, + "nodeTaints": testAccAzureRMKubernetesCluster_nodeTaints, + "nodeResourceGroup": testAccAzureRMKubernetesCluster_nodeResourceGroup, + "upgradeConfig": testAccAzureRMKubernetesCluster_upgrade, + "tags": testAccAzureRMKubernetesCluster_tags, + "windowsProfile": testAccAzureRMKubernetesCluster_windowsProfile, + "managedClusterIdentity": testAccAzureRMKubernetesCluster_managedClusterIdentiy, }, "scaling": { "addAgent": testAccAzureRMKubernetesCluster_addAgent, diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index ac3ac8c797f4..c10567f68759 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -122,6 +122,8 @@ resource "azurerm_subnet" "virtual" { * `linux_profile` - (Optional) A `linux_profile` block as defined below. +* `managed_cluster_identity` - (Optional) A `managed_cluster_identity` block as defined below. Changing this forces a new resource to be created. + * `network_profile` - (Optional) A `network_profile` block as defined below. -> **NOTE:** If `network_profile` is not defined, `kubenet` profile will be used by default. @@ -281,6 +283,12 @@ A `linux_profile` block supports the following: --- +A `mangaged_cluster_identity` block supports the following: + +* `type` - The type of identity used for the managed cluster. Valid values are `SystemAssigned` or `None`. + +--- + A `network_profile` block supports the following: * `network_plugin` - (Required) Network plugin to use for networking. Currently supported values are `azure` and `kubenet`. Changing this forces a new resource to be created. @@ -370,7 +378,7 @@ A `http_application_routing` block exports the following: --- -The `kube_admin_config` and `kube_config` blocks export the following:: +The `kube_admin_config` and `kube_config` blocks export the following: * `client_key` - Base64 encoded private key used by clients to authenticate to the Kubernetes cluster. @@ -397,6 +405,14 @@ provider "kubernetes" { } ``` +--- + +The `managed_cluster_identity` block exports the following: + +* `principal_id` - The principal id of the system assigned identity which is used by master components. + +* `tenant_id` - The tenant id of the system assigned identity which is used by master components. + ## Import Managed Kubernetes Clusters can be imported using the `resource id`, e.g. From d4589572e8eee994157467da1316195699990b2b Mon Sep 17 00:00:00 2001 From: Matthew Frahry Date: Mon, 16 Dec 2019 09:12:16 -0800 Subject: [PATCH 3/5] gosimple --- azurerm/resource_arm_kubernetes_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurerm/resource_arm_kubernetes_cluster.go b/azurerm/resource_arm_kubernetes_cluster.go index 6157245cf332..98d9c474a101 100644 --- a/azurerm/resource_arm_kubernetes_cluster.go +++ b/azurerm/resource_arm_kubernetes_cluster.go @@ -1592,7 +1592,7 @@ func flattenKubernetesClusterManagedClusterIdentity(input *containerservice.Mana return []interface{}{} } - identity := make(map[string]interface{}, 0) + identity := make(map[string]interface{}) if input.PrincipalID != nil { identity["principal_id"] = *input.PrincipalID } From b9a00047871563f549e9318f724fa98fa897f278 Mon Sep 17 00:00:00 2001 From: kt Date: Tue, 17 Dec 2019 23:58:28 -0800 Subject: [PATCH 4/5] Update website/docs/r/kubernetes_cluster.html.markdown Co-Authored-By: Tracy P Holmes <12778804+tracypholmes@users.noreply.github.com> --- website/docs/r/kubernetes_cluster.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index c10567f68759..3a8a9bc3702f 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -283,7 +283,7 @@ A `linux_profile` block supports the following: --- -A `mangaged_cluster_identity` block supports the following: +A `managed_cluster_identity` block supports the following: * `type` - The type of identity used for the managed cluster. Valid values are `SystemAssigned` or `None`. From 333e8e13383d96dcc8163273aa08bb22ca483487 Mon Sep 17 00:00:00 2001 From: Matthew Frahry Date: Wed, 18 Dec 2019 00:00:16 -0800 Subject: [PATCH 5/5] Addressing review --- azurerm/resource_arm_kubernetes_cluster.go | 17 +++++++++-------- website/docs/r/kubernetes_cluster.html.markdown | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/azurerm/resource_arm_kubernetes_cluster.go b/azurerm/resource_arm_kubernetes_cluster.go index 98d9c474a101..43b171c6abc5 100644 --- a/azurerm/resource_arm_kubernetes_cluster.go +++ b/azurerm/resource_arm_kubernetes_cluster.go @@ -827,8 +827,7 @@ func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{} if d.HasChange("managed_cluster_identity") { updateCluster = true managedClusterIdentityRaw := d.Get("managed_cluster_identity").([]interface{}) - managedClusterIdentity := expandKubernetesClusterManagedClusterIdentity(managedClusterIdentityRaw) - existing.Identity = managedClusterIdentity + existing.Identity = expandKubernetesClusterManagedClusterIdentity(managedClusterIdentityRaw) } if updateCluster { @@ -1450,14 +1449,11 @@ func expandKubernetesClusterManagedClusterIdentity(input []interface{}) *contain if len(input) == 0 || input[0] == nil { return nil } + values := input[0].(map[string]interface{}) - val := input[0].(map[string]interface{}) - - managedClusterIdentity := &containerservice.ManagedClusterIdentity{ - Type: containerservice.ResourceIdentityType(val["type"].(string)), + return &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(values["type"].(string)), } - - return managedClusterIdentity } func flattenKubernetesClusterRoleBasedAccessControl(input *containerservice.ManagedClusterProperties, d *schema.ResourceData) []interface{} { @@ -1593,12 +1589,17 @@ func flattenKubernetesClusterManagedClusterIdentity(input *containerservice.Mana } identity := make(map[string]interface{}) + + identity["principal_id"] = "" if input.PrincipalID != nil { identity["principal_id"] = *input.PrincipalID } + + identity["tenant_id"] = "" if input.TenantID != nil { identity["tenant_id"] = *input.TenantID } + identity["type"] = string(input.Type) return []interface{}{identity} diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index c10567f68759..3a8a9bc3702f 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -283,7 +283,7 @@ A `linux_profile` block supports the following: --- -A `mangaged_cluster_identity` block supports the following: +A `managed_cluster_identity` block supports the following: * `type` - The type of identity used for the managed cluster. Valid values are `SystemAssigned` or `None`.