Skip to content

Commit

Permalink
azurerm_kubernetes_cluster - support for the `azure_monitor_kuberne…
Browse files Browse the repository at this point in the history 
…tes_metrics` property (hashicorp#19530)

* `azurerm_kubernetes_cluster` - `azure_monitor_profile`

* change schema

* fmt

* rename fields

* fix minor issues in docs

Co-authored-by: Steph <steph@hashicorp.com>
  • Loading branch information
@ms-henglu @stephybun @favoretti
2 people authored and favoretti committed Jan 12, 2023
1 parent 0103987 commit b050295
Show file tree
Hide file tree
Showing 3 changed files with 229 additions and 0 deletions.
127 changes: 127 additions & 0 deletions internal/services/containers/kubernetes_cluster_other_resource_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,35 @@ func TestAccKubernetesCluster_webAppRouting(t *testing.T) {
})
}

func TestAccKubernetesCluster_azureMonitorKubernetesMetrics(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
r := KubernetesClusterResource{}

data.ResourceTest(t, r, []acceptance.TestStep{
{
Config: r.azureMonitorKubernetesMetricsEnabled(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
{
Config: r.azureMonitorKubernetesMetricsComplete(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
{
Config: r.azureMonitorKubernetesMetricsDisabled(data),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
})
}

func (KubernetesClusterResource) basicAvailabilitySetConfig(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
Expand Down Expand Up @@ -2303,3 +2332,101 @@ resource "azurerm_kubernetes_cluster" "test" {
}
`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled)
}

func (KubernetesClusterResource) azureMonitorKubernetesMetricsEnabled(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}
resource "azurerm_resource_group" "test" {
name = "acctestRG-aks-%[2]d"
location = "%[1]s"
}
resource "azurerm_kubernetes_cluster" "test" {
name = "acctestaks%[2]d"
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name
dns_prefix = "acctestaks%[2]d"
default_node_pool {
name = "default"
node_count = 1
vm_size = "Standard_DS2_v2"
}
identity {
type = "SystemAssigned"
}
monitor_metrics {
}
}
`, data.Locations.Primary, data.RandomInteger)
}

func (KubernetesClusterResource) azureMonitorKubernetesMetricsComplete(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}
resource "azurerm_resource_group" "test" {
name = "acctestRG-aks-%[2]d"
location = "%[1]s"
}
resource "azurerm_kubernetes_cluster" "test" {
name = "acctestaks%[2]d"
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name
dns_prefix = "acctestaks%[2]d"
default_node_pool {
name = "default"
node_count = 1
vm_size = "Standard_DS2_v2"
}
identity {
type = "SystemAssigned"
}
monitor_metrics {
annotations_allowed = "pods=[k8s-annotation-1,k8s-annotation-n]"
labels_allowed = "namespaces=[k8s-label-1,k8s-label-n]"
}
}
`, data.Locations.Primary, data.RandomInteger)
}

func (KubernetesClusterResource) azureMonitorKubernetesMetricsDisabled(data acceptance.TestData) string {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}
resource "azurerm_resource_group" "test" {
name = "acctestRG-aks-%[2]d"
location = "%[1]s"
}
resource "azurerm_kubernetes_cluster" "test" {
name = "acctestaks%[2]d"
location = azurerm_resource_group.test.location
resource_group_name = azurerm_resource_group.test.name
dns_prefix = "acctestaks%[2]d"
default_node_pool {
name = "default"
node_count = 1
vm_size = "Standard_DS2_v2"
}
identity {
type = "SystemAssigned"
}
}
`, data.Locations.Primary, data.RandomInteger)
}
90 changes: 90 additions & 0 deletions internal/services/containers/kubernetes_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,27 @@ func resourceKubernetesCluster() *pluginsdk.Resource {
},
},

"monitor_metrics": {
Type: pluginsdk.TypeList,
MaxItems: 1,
Optional: true,
Elem: &pluginsdk.Resource{
Schema: map[string]*pluginsdk.Schema{
"annotations_allowed": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},

"labels_allowed": {
Type: pluginsdk.TypeString,
Optional: true,
ValidateFunc: validation.StringIsNotEmpty,
},
},
},
},

"default_node_pool": SchemaDefaultNodePool(),

"disk_encryption_set_id": {
Expand Down Expand Up @@ -1243,6 +1264,9 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
autoScalerProfileRaw := d.Get("auto_scaler_profile").([]interface{})
autoScalerProfile := expandKubernetesClusterAutoScalerProfile(autoScalerProfileRaw)

azureMonitorKubernetesMetricsRaw := d.Get("monitor_metrics").([]interface{})
azureMonitorProfile := expandKubernetesClusterAzureMonitorProfile(azureMonitorKubernetesMetricsRaw)

httpProxyConfigRaw := d.Get("http_proxy_config").([]interface{})
httpProxyConfig := expandKubernetesClusterHttpProxyConfig(httpProxyConfigRaw)

Expand Down Expand Up @@ -1303,6 +1327,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
AddonProfiles: addonProfiles,
AgentPoolProfiles: agentProfiles,
AutoScalerProfile: autoScalerProfile,
AzureMonitorProfile: azureMonitorProfile,
DnsPrefix: utils.String(dnsPrefix),
EnableRBAC: utils.Bool(d.Get("role_based_access_control_enabled").(bool)),
KubernetesVersion: utils.String(kubernetesVersion),
Expand Down Expand Up @@ -1572,6 +1597,14 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{}
existing.Model.Properties.AutoScalerProfile = autoScalerProfile
}

if d.HasChange("monitor_metrics") {
updateCluster = true
azureMonitorKubernetesMetricsRaw := d.Get("monitor_metrics").([]interface{})

azureMonitorProfile := expandKubernetesClusterAzureMonitorProfile(azureMonitorKubernetesMetricsRaw)
existing.Model.Properties.AzureMonitorProfile = azureMonitorProfile
}

if d.HasChange("enable_pod_security_policy") && d.Get("enable_pod_security_policy").(bool) {
return fmt.Errorf("The AKS API has removed support for this field on 2020-10-15 and is no longer possible to configure this the Pod Security Policy - as such you'll need to set `enable_pod_security_policy` to `false`")
}
Expand Down Expand Up @@ -2052,6 +2085,11 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{})
return fmt.Errorf("setting `auto_scaler_profile`: %+v", err)
}

azureMonitorProfile := flattenKubernetesClusterAzureMonitorProfile(props.AzureMonitorProfile)
if err := d.Set("monitor_metrics", azureMonitorProfile); err != nil {
return fmt.Errorf("setting `monitor_metrics`: %+v", err)
}

flattenedDefaultNodePool, err := FlattenDefaultNodePool(props.AgentPoolProfiles, d)
if err != nil {
return fmt.Errorf("flattening `default_node_pool`: %+v", err)
Expand Down Expand Up @@ -3498,3 +3536,55 @@ func flattenKubernetesClusterIngressProfile(input *managedclusters.ManagedCluste
},
}
}

func expandKubernetesClusterAzureMonitorProfile(input []interface{}) *managedclusters.ManagedClusterAzureMonitorProfile {
if len(input) == 0 {
return &managedclusters.ManagedClusterAzureMonitorProfile{
Metrics: &managedclusters.ManagedClusterAzureMonitorProfileMetrics{
Enabled: false,
},
}
}
if input[0] == nil {
return &managedclusters.ManagedClusterAzureMonitorProfile{
Metrics: &managedclusters.ManagedClusterAzureMonitorProfileMetrics{
Enabled: true,
},
}
}
config := input[0].(map[string]interface{})
return &managedclusters.ManagedClusterAzureMonitorProfile{
Metrics: &managedclusters.ManagedClusterAzureMonitorProfileMetrics{
Enabled: true,
KubeStateMetrics: &managedclusters.ManagedClusterAzureMonitorProfileKubeStateMetrics{
MetricAnnotationsAllowList: utils.String(config["annotations_allowed"].(string)),
MetricLabelsAllowlist: utils.String(config["labels_allowed"].(string)),
},
},
}
}

func flattenKubernetesClusterAzureMonitorProfile(input *managedclusters.ManagedClusterAzureMonitorProfile) []interface{} {
if input == nil || input.Metrics == nil || !input.Metrics.Enabled {
return nil
}
if input.Metrics.KubeStateMetrics == nil {
return []interface{}{
map[string]interface{}{},
}
}
annotationAllowList := ""
if input.Metrics.KubeStateMetrics.MetricAnnotationsAllowList != nil {
annotationAllowList = *input.Metrics.KubeStateMetrics.MetricAnnotationsAllowList
}
labelAllowList := ""
if input.Metrics.KubeStateMetrics.MetricLabelsAllowlist != nil {
labelAllowList = *input.Metrics.KubeStateMetrics.MetricLabelsAllowlist
}
return []interface{}{
map[string]interface{}{
"annotations_allowed": annotationAllowList,
"labels_allowed": labelAllowList,
},
}
}
12 changes: 12 additions & 0 deletions website/docs/r/kubernetes_cluster.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ In addition, one of either `identity` or `service_principal` blocks must be spec

* `azure_active_directory_role_based_access_control` - (Optional) A `azure_active_directory_role_based_access_control` block as defined below.

* `monitor_metrics` - (Optional) Specifies a Prometheus add-on profile for the Kubernetes Cluster. A `monitor_metrics` block as defined below.

-> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/AKS-PrometheusAddonPreview` is enabled, see [the documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-metrics-enable?tabs=azure-portal) for more information.

* `azure_policy_enabled` - (Optional) Should the Azure Policy Add-On be enabled? For more details please visit [Understand Azure Policy for Azure Kubernetes Service](https://docs.microsoft.com/en-ie/azure/governance/policy/concepts/rego-for-aks)

* `disk_encryption_set_id` - (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created.
Expand Down Expand Up @@ -324,6 +328,14 @@ When `managed` is set to `false` the following properties can be specified:

---

An `monitor_metrics` block supports the following:

* `annotations_allowed` - (Optional) Specifies a comma-separated list of Kubernetes annotation keys that will be used in the resource's labels metric.

* `annotations_allowed` - (Optional) Specifies a Comma-separated list of additional Kubernetes label keys that will be used in the resource's labels metric.

---

A `default_node_pool` block supports the following:

* `name` - (Required) The name which should be used for the default Kubernetes Node Pool. Changing this forces a new resource to be created.
Expand Down

0 comments on commit b050295

Please sign in to comment.