From eeb3f4fed28e14d73346ed45d33d9bbfcd445ac0 Mon Sep 17 00:00:00 2001 From: Tolsee Date: Sat, 14 Jan 2023 07:01:55 +0545 Subject: [PATCH] feat!: promote gke_backup_agent_config to ga (#1513) --- README.md | 3 ++- autogen/main/cluster.tf.tmpl | 24 ++++++++++++------- autogen/main/main.tf.tmpl | 4 ++-- autogen/main/variables.tf.tmpl | 16 ++++++++----- autogen/main/versions.tf.tmpl | 2 +- autogen/safer-cluster/variables.tf.tmpl | 4 ++++ cluster.tf | 16 +++++++++++++ main.tf | 1 + .../versions.tf | 2 +- .../beta-autopilot-public-cluster/versions.tf | 2 +- .../README.md | 4 ++-- .../cluster.tf | 24 ++++++++++++------- .../main.tf | 2 +- .../variables.tf | 16 ++++++++----- .../versions.tf | 2 +- modules/beta-private-cluster/README.md | 4 ++-- modules/beta-private-cluster/cluster.tf | 24 ++++++++++++------- modules/beta-private-cluster/main.tf | 2 +- modules/beta-private-cluster/variables.tf | 16 ++++++++----- modules/beta-private-cluster/versions.tf | 2 +- .../README.md | 4 ++-- .../cluster.tf | 24 ++++++++++++------- .../main.tf | 2 +- .../variables.tf | 16 ++++++++----- .../versions.tf | 2 +- modules/beta-public-cluster/README.md | 4 ++-- modules/beta-public-cluster/cluster.tf | 24 ++++++++++++------- modules/beta-public-cluster/main.tf | 2 +- modules/beta-public-cluster/variables.tf | 16 ++++++++----- modules/beta-public-cluster/versions.tf | 2 +- .../private-cluster-update-variant/README.md | 3 ++- .../private-cluster-update-variant/cluster.tf | 16 +++++++++++++ .../private-cluster-update-variant/main.tf | 1 + .../variables.tf | 10 ++++++++ modules/private-cluster/README.md | 3 ++- modules/private-cluster/cluster.tf | 16 +++++++++++++ modules/private-cluster/main.tf | 1 + modules/private-cluster/variables.tf | 10 ++++++++ .../safer-cluster-update-variant/README.md | 2 +- .../safer-cluster-update-variant/variables.tf | 4 ++++ modules/safer-cluster/README.md | 2 +- modules/safer-cluster/variables.tf | 4 ++++ .../testdata/TestDisableClientCert.json | 1 + .../TestPrivateZonalWithNetworking.json | 1 + variables.tf | 10 ++++++++ 45 files changed, 254 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 4e34f2c6bb..d11869a0ae 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -158,6 +158,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 67f30694db..6ed56533f8 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -151,6 +151,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } {% endif %} {% if autopilot_cluster == true %} @@ -259,6 +267,14 @@ resource "google_container_cluster" "primary" { enabled = gce_persistent_disk_csi_driver_config.value.enabled } } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } {% endif %} {% if beta_cluster and autopilot_cluster != true %} @@ -282,14 +298,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } {% endif %} } diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index cb637d58a7..5f7b1df381 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -68,7 +68,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }], var.cluster_autoscaling.gpu_resources) : [] + }], var.cluster_autoscaling.gpu_resources) : [] {% endif %} @@ -103,6 +103,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] {% endif %} {% if beta_cluster and autopilot_cluster != true %} cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { @@ -117,7 +118,6 @@ locals { ) ] : [] cluster_cloudrun_enabled = var.cloudrun - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] {% endif %} cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index ff3c153b82..d82622ea3a 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -241,6 +241,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -252,6 +254,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -641,6 +645,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + {% endif %} variable "timeouts" { type = map(string) @@ -703,12 +713,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 2c36fce02b..50f1c956b3 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -24,7 +24,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index a357cbe241..5344f81b06 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -227,6 +227,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -236,6 +238,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/cluster.tf b/cluster.tf index e0fa17b8af..1833f66575 100644 --- a/cluster.tf +++ b/cluster.tf @@ -114,6 +114,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -184,6 +192,14 @@ resource "google_container_cluster" "primary" { enabled = gce_persistent_disk_csi_driver_config.value.enabled } } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } } datapath_provider = var.datapath_provider diff --git a/main.tf b/main.tf index 41a6e1c3c9..48bf66213a 100644 --- a/main.tf +++ b/main.tf @@ -85,6 +85,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf index e0d011de5f..0e367f0344 100644 --- a/modules/beta-autopilot-private-cluster/versions.tf +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf index 5c1a49a6ad..6df1955473 100644 --- a/modules/beta-autopilot-public-cluster/versions.tf +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 9bb5cf1300..872ceaae06 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -165,7 +165,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -203,7 +203,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 35432586e3..3394747654 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -128,6 +128,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -217,6 +225,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -237,14 +253,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 73521f01ec..975390944e 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -86,6 +86,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -98,7 +99,6 @@ locals { ) ] : [] cluster_cloudrun_enabled = var.cloudrun - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index a1c33bb503..7fa57dfe6d 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -234,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -243,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -614,6 +618,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -671,12 +681,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index ef70d94a7c..d578a80209 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 6f46c9ae45..7668f6ea08 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -143,7 +143,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -181,7 +181,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 7b5425c9df..0e5153cee7 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -128,6 +128,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -217,6 +225,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -237,14 +253,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 73521f01ec..975390944e 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -86,6 +86,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -98,7 +99,6 @@ locals { ) ] : [] cluster_cloudrun_enabled = var.cloudrun - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index a1c33bb503..7fa57dfe6d 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -234,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -243,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -614,6 +618,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -671,12 +681,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index b7ba568fcd..2fc9d7c76f 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 58e14ec261..08dc370ebc 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -159,7 +159,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -194,7 +194,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 8480a0bfbb..6b5f0d93f7 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -128,6 +128,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -217,6 +225,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -237,14 +253,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index eac0840002..491200ba7e 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -86,6 +86,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -98,7 +99,6 @@ locals { ) ] : [] cluster_cloudrun_enabled = var.cloudrun - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 85dfbd4325..970d25f40d 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -234,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -243,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -584,6 +588,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -641,12 +651,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index 8424866415..a4d5064d59 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index fd174ff940..8988a490b2 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,7 +137,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -172,7 +172,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 904a53c8a2..587348ead3 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -128,6 +128,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -217,6 +225,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -237,14 +253,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index eac0840002..491200ba7e 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -86,6 +86,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -98,7 +99,6 @@ locals { ) ] : [] cluster_cloudrun_enabled = var.cloudrun - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 85dfbd4325..970d25f40d 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -234,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -243,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -584,6 +588,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -641,12 +651,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index 147446d22a..413ce1cf07 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.46.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index ed0579ab0e..ea7035c14f 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -159,7 +159,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -189,6 +189,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 48775f35b0..353e908f94 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -114,6 +114,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -184,6 +192,14 @@ resource "google_container_cluster" "primary" { enabled = gce_persistent_disk_csi_driver_config.value.enabled } } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } } datapath_provider = var.datapath_provider diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 4f9a4f7ab2..1082e1abca 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -85,6 +85,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index c2deafdfed..240e128df6 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -222,6 +222,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -230,6 +232,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -585,6 +589,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index c0f5d07acb..ba944d0e14 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -137,7 +137,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -167,6 +167,7 @@ Then perform the following commands on the root folder: | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index f5ffb39fbd..48e0408811 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -114,6 +114,14 @@ resource "google_container_cluster" "primary" { maximum = lookup(resource_limits.value, "maximum") } } + dynamic "management" { + for_each = var.cluster_autoscaling.enabled ? [1] : [] + + content { + auto_repair = var.cluster_autoscaling.auto_repair + auto_upgrade = var.cluster_autoscaling.auto_upgrade + } + } } vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -184,6 +192,14 @@ resource "google_container_cluster" "primary" { enabled = gce_persistent_disk_csi_driver_config.value.enabled } } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } } datapath_provider = var.datapath_provider diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 4f9a4f7ab2..1082e1abca 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -85,6 +85,7 @@ locals { }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index c2deafdfed..240e128df6 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -222,6 +222,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -230,6 +232,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -585,6 +589,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index 51875903a4..65f77cdf95 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -204,7 +204,7 @@ For simplicity, we suggest using `roles/container.admin` and | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index f42929b53f..07430d6f78 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -227,6 +227,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -236,6 +238,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 51875903a4..65f77cdf95 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -204,7 +204,7 @@ For simplicity, we suggest using `roles/container.admin` and | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index f42929b53f..07430d6f78 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -227,6 +227,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -236,6 +238,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/test/integration/disable_client_cert/testdata/TestDisableClientCert.json b/test/integration/disable_client_cert/testdata/TestDisableClientCert.json index 3128aac00a..9d58f326f8 100755 --- a/test/integration/disable_client_cert/testdata/TestDisableClientCert.json +++ b/test/integration/disable_client_cert/testdata/TestDisableClientCert.json @@ -5,6 +5,7 @@ "enabled": true }, "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, "horizontalPodAutoscaling": {}, "httpLoadBalancing": {}, "kubernetesDashboard": { diff --git a/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json b/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json index d1871d8bad..b2b0009907 100755 --- a/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json +++ b/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json @@ -5,6 +5,7 @@ "enabled": true }, "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, "horizontalPodAutoscaling": {}, "httpLoadBalancing": {}, "kubernetesDashboard": { diff --git a/variables.tf b/variables.tf index 923b7215d2..2efed7a228 100644 --- a/variables.tf +++ b/variables.tf @@ -222,6 +222,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -230,6 +232,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -555,6 +559,12 @@ variable "gce_pd_csi_driver" { default = true } +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations."