azurerm_kubernetes_cluster - support for the azure_monitor_kubernetes_metrics property

internal/services/containers/kubernetes_cluster_other_resource_test.go

@@ -705,6 +705,21 @@ func TestAccKubernetesCluster_webAppRouting(t *testing.T) {
 	})
 }
 
+func TestAccKubernetesCluster_azureMonitorProfile(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
+	r := KubernetesClusterResource{}
+
+	data.ResourceTest(t, r, []acceptance.TestStep{
+		{
+			Config: r.azureMonitorProfile(data),
+			Check: acceptance.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+			),
+		},
+		data.ImportStep(),
+	})
+}
+
 func (KubernetesClusterResource) basicAvailabilitySetConfig(data acceptance.TestData) string {
 	return fmt.Sprintf(`
 provider "azurerm" {
@@ -2254,3 +2269,43 @@ resource "azurerm_kubernetes_cluster" "test" {
 }
  `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger)
 }
+
+func (KubernetesClusterResource) azureMonitorProfile(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+provider "azurerm" {
+  features {}
+}
+
+resource "azurerm_resource_group" "test" {
+  name     = "acctestRG-aks-%[2]d"
+  location = "%[1]s"
+}
+
+resource "azurerm_kubernetes_cluster" "test" {
+  name                = "acctestaks%[2]d"
+  location            = azurerm_resource_group.test.location
+  resource_group_name = azurerm_resource_group.test.name
+  dns_prefix          = "acctestaks%[2]d"
+
+  default_node_pool {
+    name       = "default"
+    node_count = 1
+    vm_size    = "Standard_DS2_v2"
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  azure_monitor_profile {
+    metrics {
+      enabled = true
+      kube_state_metrics {
+        metric_annotations_allow_list = "pods=[k8s-annotation-1,k8s-annotation-n]"
+        metric_labels_allow_list      = "namespaces=[k8s-label-1,k8s-label-n]"
+      }
+    }
+  }
+}
+ `, data.Locations.Primary, data.RandomInteger)
+}

internal/services/containers/kubernetes_cluster_resource.go

@@ -307,6 +307,50 @@ func resourceKubernetesCluster() *pluginsdk.Resource {
 				},
 			},
 
+			"azure_monitor_profile": {
+				Type:     pluginsdk.TypeList,
+				Optional: true,
+				MaxItems: 1,
+				Elem: &pluginsdk.Resource{
+					Schema: map[string]*pluginsdk.Schema{
+						"metrics": {
+							Type:     pluginsdk.TypeList,
+							MaxItems: 1,
+							Optional: true,
+							Elem: &pluginsdk.Resource{
+								Schema: map[string]*pluginsdk.Schema{
+									"enabled": {
+										Type:     pluginsdk.TypeBool,
+										Required: true,
+									},
+
+									"kube_state_metrics": {
+										Type:     pluginsdk.TypeList,
+										MaxItems: 1,
+										Optional: true,
+										Elem: &pluginsdk.Resource{
+											Schema: map[string]*pluginsdk.Schema{
+												"metric_annotations_allow_list": {
+													Type:         pluginsdk.TypeString,
+													Optional:     true,
+													ValidateFunc: validation.StringIsNotEmpty,
+												},
+
+												"metric_labels_allow_list": {
+													Type:         pluginsdk.TypeString,
+													Optional:     true,
+													ValidateFunc: validation.StringIsNotEmpty,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+
 			"default_node_pool": SchemaDefaultNodePool(),
 
 			"disk_encryption_set_id": {
@@ -1190,6 +1234,9 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
 	autoScalerProfileRaw := d.Get("auto_scaler_profile").([]interface{})
 	autoScalerProfile := expandKubernetesClusterAutoScalerProfile(autoScalerProfileRaw)
 
+	azureMonitorProfileRaw := d.Get("azure_monitor_profile").([]interface{})
+	azureMonitorProfile := expandKubernetesClusterAzureMonitorProfile(azureMonitorProfileRaw)
+
 	httpProxyConfigRaw := d.Get("http_proxy_config").([]interface{})
 	httpProxyConfig := expandKubernetesClusterHttpProxyConfig(httpProxyConfigRaw)
 
@@ -1239,6 +1286,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
 			AddonProfiles:             addonProfiles,
 			AgentPoolProfiles:         agentProfiles,
 			AutoScalerProfile:         autoScalerProfile,
+			AzureMonitorProfile:       azureMonitorProfile,
 			DnsPrefix:                 utils.String(dnsPrefix),
 			EnableRBAC:                utils.Bool(d.Get("role_based_access_control_enabled").(bool)),
 			KubernetesVersion:         utils.String(kubernetesVersion),
@@ -1508,6 +1556,14 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{}
 		existing.Model.Properties.AutoScalerProfile = autoScalerProfile
 	}
 
+	if d.HasChange("azure_monitor_profile") {
+		updateCluster = true
+		azureMonitorProfileRaw := d.Get("azure_monitor_profile").([]interface{})
+
+		azureMonitorProfile := expandKubernetesClusterAzureMonitorProfile(azureMonitorProfileRaw)
+		existing.Model.Properties.AzureMonitorProfile = azureMonitorProfile
+	}
+
 	if d.HasChange("enable_pod_security_policy") && d.Get("enable_pod_security_policy").(bool) {
 		return fmt.Errorf("The AKS API has removed support for this field on 2020-10-15 and is no longer possible to configure this the Pod Security Policy - as such you'll need to set `enable_pod_security_policy` to `false`")
 	}
@@ -1970,6 +2026,11 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{})
 			return fmt.Errorf("setting `auto_scaler_profile`: %+v", err)
 		}
 
+		azureMonitorProfile := flattenKubernetesClusterAzureMonitorProfile(props.AzureMonitorProfile)
+		if err := d.Set("azure_monitor_profile", azureMonitorProfile); err != nil {
+			return fmt.Errorf("setting `azure_monitor_profile`: %+v", err)
+		}
+
 		flattenedDefaultNodePool, err := FlattenDefaultNodePool(props.AgentPoolProfiles, d)
 		if err != nil {
 			return fmt.Errorf("flattening `default_node_pool`: %+v", err)
@@ -3385,3 +3446,86 @@ func flattenKubernetesClusterIngressProfile(input *managedclusters.ManagedCluste
 		},
 	}
 }
+
+func expandKubernetesClusterAzureMonitorProfile(input []interface{}) *managedclusters.ManagedClusterAzureMonitorProfile {
+	if len(input) == 0 || input[0] == nil {
+		return nil
+	}
+	config := input[0].(map[string]interface{})
+	return &managedclusters.ManagedClusterAzureMonitorProfile{
+		Metrics: expandKubernetesClusterAzureMonitorProfileMetrics(config["metrics"]),
+	}
+}
+
+func expandKubernetesClusterAzureMonitorProfileMetrics(raw interface{}) *managedclusters.ManagedClusterAzureMonitorProfileMetrics {
+	if raw == nil {
+		return nil
+	}
+	input := raw.([]interface{})
+	if len(input) == 0 || input[0] == nil {
+		return nil
+	}
+	config := input[0].(map[string]interface{})
+	return &managedclusters.ManagedClusterAzureMonitorProfileMetrics{
+		Enabled:          config["enabled"].(bool),
+		KubeStateMetrics: expandKubernetesClusterAzureMonitorProfileKubeStateMetrics(config["kube_state_metrics"]),
+	}
+}
+
+func expandKubernetesClusterAzureMonitorProfileKubeStateMetrics(raw interface{}) *managedclusters.ManagedClusterAzureMonitorProfileKubeStateMetrics {
+	if raw == nil {
+		return nil
+	}
+	input := raw.([]interface{})
+	if len(input) == 0 || input[0] == nil {
+		return nil
+	}
+	config := input[0].(map[string]interface{})
+	return &managedclusters.ManagedClusterAzureMonitorProfileKubeStateMetrics{
+		MetricAnnotationsAllowList: utils.String(config["metric_annotations_allow_list"].(string)),
+		MetricLabelsAllowlist:      utils.String(config["metric_labels_allow_list"].(string)),
+	}
+}
+
+func flattenKubernetesClusterAzureMonitorProfile(input *managedclusters.ManagedClusterAzureMonitorProfile) []interface{} {
+	if input == nil {
+		return nil
+	}
+	return []interface{}{
+		map[string]interface{}{
+			"metrics": flattenKubernetesClusterAzureMonitorProfileMetrics(input.Metrics),
+		},
+	}
+}
+
+func flattenKubernetesClusterAzureMonitorProfileMetrics(input *managedclusters.ManagedClusterAzureMonitorProfileMetrics) []interface{} {
+	if input == nil {
+		return nil
+	}
+	return []interface{}{
+		map[string]interface{}{
+			"enabled":            input.Enabled,
+			"kube_state_metrics": flattenKubernetesClusterAzureMonitorProfileKubeStateMetrics(input.KubeStateMetrics),
+		},
+	}
+}
+
+func flattenKubernetesClusterAzureMonitorProfileKubeStateMetrics(input *managedclusters.ManagedClusterAzureMonitorProfileKubeStateMetrics) []interface{} {
+	if input == nil {
+		return nil
+	}
+	annotationAllowList := ""
+	if input.MetricAnnotationsAllowList != nil {
+		annotationAllowList = *input.MetricAnnotationsAllowList
+	}
+	labelAllowList := ""
+	if input.MetricLabelsAllowlist != nil {
+		labelAllowList = *input.MetricLabelsAllowlist
+	}
+	return []interface{}{
+		map[string]interface{}{
+			"metric_annotations_allow_list": annotationAllowList,
+			"metric_labels_allow_list":      labelAllowList,
+		},
+	}
+}

website/docs/r/kubernetes_cluster.html.markdown

@@ -97,6 +97,10 @@ In addition, one of either `identity` or `service_principal` blocks must be spec
 
 * `azure_active_directory_role_based_access_control` - (Optional) - A `azure_active_directory_role_based_access_control` block as defined below.
 
+* `azure_monitor_profile` - (Optional) Specifies prometheus addon profile for the container service cluster. A `azure_monitor_profile` block as defined below.
+
+-> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/AKS-PrometheusAddonPreview` is enabled, see [the documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-metrics-enable?tabs=azure-portal) for more information.
+
 * `azure_policy_enabled` - (Optional) Should the Azure Policy Add-On be enabled? For more details please visit [Understand Azure Policy for Azure Kubernetes Service](https://docs.microsoft.com/en-ie/azure/governance/policy/concepts/rego-for-aks)
 
 * `disk_encryption_set_id` - (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created.
@@ -316,6 +320,28 @@ When `managed` is set to `false` the following properties can be specified:
 
 ---
 
+An `azure_monitor_profile` block supports the following:
+
+* `metrics` - (Optional) Specifies Metrics profile for the prometheus service addon. A `metrics` block as defined below.
+
+---
+
+An `metrics` block supports the following:
+
+* `enabled` - (Required) Should enable the Prometheus collector?
+
+* `kube_state_metrics` - (Optional) Specifies the Kube State Metrics for prometheus addon profile for the container service cluster. A `kube_state_metrics` block as defined below.
+
+---
+
+An `kube_state_metrics` block supports the following:
+
+* `metric_annotations_allow_list` - (Optional) Specifies a comma-separated list of Kubernetes annotations keys that will be used in the resource's labels metric.
+
+* `metric_annotations_allow_list` - (Optional) Specifies a Comma-separated list of additional Kubernetes label keys that will be used in the resource's labels metric.
+
+---
+
 A `default_node_pool` block supports the following:
 
 * `name` - (Required) The name which should be used for the default Kubernetes Node Pool. Changing this forces a new resource to be created.