diff --git a/internal/services/containers/kubernetes_cluster_other_resource_test.go b/internal/services/containers/kubernetes_cluster_other_resource_test.go index 9dda103c1337..cd7a1a7e2fca 100644 --- a/internal/services/containers/kubernetes_cluster_other_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_other_resource_test.go @@ -727,6 +727,35 @@ func TestAccKubernetesCluster_webAppRouting(t *testing.T) { }) } +func TestAccKubernetesCluster_azureMonitorKubernetesMetrics(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.azureMonitorKubernetesMetricsEnabled(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + { + Config: r.azureMonitorKubernetesMetricsComplete(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + { + Config: r.azureMonitorKubernetesMetricsDisabled(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func (KubernetesClusterResource) basicAvailabilitySetConfig(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { @@ -2303,3 +2332,101 @@ resource "azurerm_kubernetes_cluster" "test" { } `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled) } + +func (KubernetesClusterResource) azureMonitorKubernetesMetricsEnabled(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%[2]d" + location = "%[1]s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%[2]d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%[2]d" + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "SystemAssigned" + } + + monitor_metrics { + } +} + `, data.Locations.Primary, data.RandomInteger) +} + +func (KubernetesClusterResource) azureMonitorKubernetesMetricsComplete(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%[2]d" + location = "%[1]s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%[2]d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%[2]d" + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "SystemAssigned" + } + + monitor_metrics { + annotations_allowed = "pods=[k8s-annotation-1,k8s-annotation-n]" + labels_allowed = "namespaces=[k8s-label-1,k8s-label-n]" + } +} + `, data.Locations.Primary, data.RandomInteger) +} + +func (KubernetesClusterResource) azureMonitorKubernetesMetricsDisabled(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%[2]d" + location = "%[1]s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%[2]d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%[2]d" + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "SystemAssigned" + } +} + `, data.Locations.Primary, data.RandomInteger) +} diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 3113c0417c90..b2858d136614 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -307,6 +307,27 @@ func resourceKubernetesCluster() *pluginsdk.Resource { }, }, + "monitor_metrics": { + Type: pluginsdk.TypeList, + MaxItems: 1, + Optional: true, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "annotations_allowed": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: validation.StringIsNotEmpty, + }, + + "labels_allowed": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: validation.StringIsNotEmpty, + }, + }, + }, + }, + "default_node_pool": SchemaDefaultNodePool(), "disk_encryption_set_id": { @@ -1261,6 +1282,9 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} autoScalerProfileRaw := d.Get("auto_scaler_profile").([]interface{}) autoScalerProfile := expandKubernetesClusterAutoScalerProfile(autoScalerProfileRaw) + azureMonitorKubernetesMetricsRaw := d.Get("monitor_metrics").([]interface{}) + azureMonitorProfile := expandKubernetesClusterAzureMonitorProfile(azureMonitorKubernetesMetricsRaw) + httpProxyConfigRaw := d.Get("http_proxy_config").([]interface{}) httpProxyConfig := expandKubernetesClusterHttpProxyConfig(httpProxyConfigRaw) @@ -1321,6 +1345,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} AddonProfiles: addonProfiles, AgentPoolProfiles: agentProfiles, AutoScalerProfile: autoScalerProfile, + AzureMonitorProfile: azureMonitorProfile, DnsPrefix: utils.String(dnsPrefix), EnableRBAC: utils.Bool(d.Get("role_based_access_control_enabled").(bool)), KubernetesVersion: utils.String(kubernetesVersion), @@ -1590,6 +1615,14 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} existing.Model.Properties.AutoScalerProfile = autoScalerProfile } + if d.HasChange("monitor_metrics") { + updateCluster = true + azureMonitorKubernetesMetricsRaw := d.Get("monitor_metrics").([]interface{}) + + azureMonitorProfile := expandKubernetesClusterAzureMonitorProfile(azureMonitorKubernetesMetricsRaw) + existing.Model.Properties.AzureMonitorProfile = azureMonitorProfile + } + if d.HasChange("enable_pod_security_policy") && d.Get("enable_pod_security_policy").(bool) { return fmt.Errorf("The AKS API has removed support for this field on 2020-10-15 and is no longer possible to configure this the Pod Security Policy - as such you'll need to set `enable_pod_security_policy` to `false`") } @@ -2070,6 +2103,11 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) return fmt.Errorf("setting `auto_scaler_profile`: %+v", err) } + azureMonitorProfile := flattenKubernetesClusterAzureMonitorProfile(props.AzureMonitorProfile) + if err := d.Set("monitor_metrics", azureMonitorProfile); err != nil { + return fmt.Errorf("setting `monitor_metrics`: %+v", err) + } + flattenedDefaultNodePool, err := FlattenDefaultNodePool(props.AgentPoolProfiles, d) if err != nil { return fmt.Errorf("flattening `default_node_pool`: %+v", err) @@ -3538,3 +3576,55 @@ func flattenKubernetesClusterIngressProfile(input *managedclusters.ManagedCluste }, } } + +func expandKubernetesClusterAzureMonitorProfile(input []interface{}) *managedclusters.ManagedClusterAzureMonitorProfile { + if len(input) == 0 { + return &managedclusters.ManagedClusterAzureMonitorProfile{ + Metrics: &managedclusters.ManagedClusterAzureMonitorProfileMetrics{ + Enabled: false, + }, + } + } + if input[0] == nil { + return &managedclusters.ManagedClusterAzureMonitorProfile{ + Metrics: &managedclusters.ManagedClusterAzureMonitorProfileMetrics{ + Enabled: true, + }, + } + } + config := input[0].(map[string]interface{}) + return &managedclusters.ManagedClusterAzureMonitorProfile{ + Metrics: &managedclusters.ManagedClusterAzureMonitorProfileMetrics{ + Enabled: true, + KubeStateMetrics: &managedclusters.ManagedClusterAzureMonitorProfileKubeStateMetrics{ + MetricAnnotationsAllowList: utils.String(config["annotations_allowed"].(string)), + MetricLabelsAllowlist: utils.String(config["labels_allowed"].(string)), + }, + }, + } +} + +func flattenKubernetesClusterAzureMonitorProfile(input *managedclusters.ManagedClusterAzureMonitorProfile) []interface{} { + if input == nil || input.Metrics == nil || !input.Metrics.Enabled { + return nil + } + if input.Metrics.KubeStateMetrics == nil { + return []interface{}{ + map[string]interface{}{}, + } + } + annotationAllowList := "" + if input.Metrics.KubeStateMetrics.MetricAnnotationsAllowList != nil { + annotationAllowList = *input.Metrics.KubeStateMetrics.MetricAnnotationsAllowList + } + labelAllowList := "" + if input.Metrics.KubeStateMetrics.MetricLabelsAllowlist != nil { + labelAllowList = *input.Metrics.KubeStateMetrics.MetricLabelsAllowlist + } + return []interface{}{ + map[string]interface{}{ + "annotations_allowed": annotationAllowList, + "labels_allowed": labelAllowList, + }, + } +} diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index faa3afb90ac5..462b10d3b227 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -97,6 +97,10 @@ In addition, one of either `identity` or `service_principal` blocks must be spec * `azure_active_directory_role_based_access_control` - (Optional) A `azure_active_directory_role_based_access_control` block as defined below. +* `monitor_metrics` - (Optional) Specifies a Prometheus add-on profile for the Kubernetes Cluster. A `monitor_metrics` block as defined below. + +-> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/AKS-PrometheusAddonPreview` is enabled, see [the documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-metrics-enable?tabs=azure-portal) for more information. + * `azure_policy_enabled` - (Optional) Should the Azure Policy Add-On be enabled? For more details please visit [Understand Azure Policy for Azure Kubernetes Service](https://docs.microsoft.com/en-ie/azure/governance/policy/concepts/rego-for-aks) * `disk_encryption_set_id` - (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created. @@ -324,6 +328,14 @@ When `managed` is set to `false` the following properties can be specified: --- +An `monitor_metrics` block supports the following: + +* `annotations_allowed` - (Optional) Specifies a comma-separated list of Kubernetes annotation keys that will be used in the resource's labels metric. + +* `annotations_allowed` - (Optional) Specifies a Comma-separated list of additional Kubernetes label keys that will be used in the resource's labels metric. + +--- + A `default_node_pool` block supports the following: * `name` - (Required) The name which should be used for the default Kubernetes Node Pool. Changing this forces a new resource to be created.