Moving autoscaling from beta to GA

terraform-google-modules · Oct 12, 2020 · 3cc16d3 · 3cc16d3
1 parent 6cb85fb
commit 3cc16d3
Show file tree

Hide file tree

Showing 21 changed files with 148 additions and 25 deletions.
diff --git a/README.md b/README.md
@@ -103,6 +103,7 @@ Then perform the following commands on the root folder:
 | add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `<map>` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |

diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -65,10 +65,11 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
-{% if beta_cluster %}
   cluster_autoscaling {
     enabled             = var.cluster_autoscaling.enabled
+{% if beta_cluster %}
     autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED"
+{% endif %}
     dynamic "resource_limits" {
       for_each = local.autoscalling_resource_limits
       content {
@@ -78,7 +79,6 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
-{% endif %}
 
   default_max_pods_per_node = var.default_max_pods_per_node
 
@@ -389,10 +389,8 @@ resource "google_container_node_pool" "pools" {
   {% endif %}
   project  = var.project_id
   location = local.location
-  {% if beta_cluster %}
   // use node_locations if provided, defaults to cluster level node_locations if not specified
   node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null
-  {% endif %}
 
   cluster = google_container_cluster.primary.name
 

diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl
@@ -51,7 +51,6 @@ locals {
   node_pools      = zipmap(local.node_pool_names, tolist(toset(var.node_pools)))
 
   release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
-{% if beta_cluster %}
 
   autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{
     resource_type = "cpu"
@@ -63,8 +62,6 @@ locals {
     maximum       = var.cluster_autoscaling.max_memory_gb
   }] : []
 
-{% endif %}
-
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0

diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -207,27 +207,31 @@ variable "enable_kubernetes_alpha" {
   description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days."
   default     = false
 }
+{% endif %}
 
 variable "cluster_autoscaling" {
   type = object({
     enabled             = bool
+{% if beta_cluster %}
     autoscaling_profile = string
+{% endif %}
     min_cpu_cores       = number
     max_cpu_cores       = number
     min_memory_gb       = number
     max_memory_gb       = number
   })
   default = {
     enabled             = false
+{% if beta_cluster %}
     autoscaling_profile = "BALANCED"
+{% endif %}
     max_cpu_cores       = 0
     min_cpu_cores       = 0
     max_memory_gb       = 0
     min_memory_gb       = 0
   }
   description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)"
 }
-{% endif %}
 
 variable "node_pools_taints" {
   type        = map(list(object({ key = string, value = string, effect = string })))

diff --git a/cluster.tf b/cluster.tf
@@ -56,6 +56,17 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
+  cluster_autoscaling {
+    enabled = var.cluster_autoscaling.enabled
+    dynamic "resource_limits" {
+      for_each = local.autoscalling_resource_limits
+      content {
+        resource_type = lookup(resource_limits.value, "resource_type")
+        minimum       = lookup(resource_limits.value, "minimum")
+        maximum       = lookup(resource_limits.value, "maximum")
+      }
+    }
+  }
 
   default_max_pods_per_node = var.default_max_pods_per_node
 
@@ -181,6 +192,8 @@ resource "google_container_node_pool" "pools" {
   name     = each.key
   project  = var.project_id
   location = local.location
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null
 
   cluster = google_container_cluster.primary.name
 

diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf
@@ -54,20 +54,22 @@ variable "compute_engine_service_account" {
 
 variable "cluster_autoscaling" {
   type = object({
-    enabled             = bool
-    autoscaling_profile = string
-    min_cpu_cores       = number
-    max_cpu_cores       = number
-    min_memory_gb       = number
-    max_memory_gb       = number
+    enabled = bool
+    //Enable when autoscaling_profile becomes GA
+    //autoscaling_profile = string
+    min_cpu_cores = number
+    max_cpu_cores = number
+    min_memory_gb = number
+    max_memory_gb = number
   })
   default = {
-    enabled             = false
-    autoscaling_profile = "BALANCED"
-    max_cpu_cores       = 0
-    min_cpu_cores       = 0
-    max_memory_gb       = 0
-    min_memory_gb       = 0
+    enabled = false
+    //Enable when autoscaling_profile becomes GA
+    //autoscaling_profile = "BALANCED"
+    max_cpu_cores = 0
+    min_cpu_cores = 0
+    max_memory_gb = 0
+    min_memory_gb = 0
   }
   description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)"
 }
diff --git a/main.tf b/main.tf
@@ -48,6 +48,16 @@ locals {
 
   release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
 
+  autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{
+    resource_type = "cpu"
+    minimum       = var.cluster_autoscaling.min_cpu_cores
+    maximum       = var.cluster_autoscaling.max_cpu_cores
+    }, {
+    resource_type = "memory"
+    minimum       = var.cluster_autoscaling.min_memory_gb
+    maximum       = var.cluster_autoscaling.max_memory_gb
+  }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0

diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf
@@ -59,7 +59,6 @@ locals {
   }] : []
 
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf
@@ -59,7 +59,6 @@ locals {
   }] : []
 
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf
@@ -59,7 +59,6 @@ locals {
   }] : []
 
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf
@@ -59,7 +59,6 @@ locals {
   }] : []
 
 
-
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0
   network_project_id          = var.network_project_id != "" ? var.network_project_id : var.project_id

diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md
@@ -131,6 +131,7 @@ Then perform the following commands on the root folder:
 | add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `<map>` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |

diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf
@@ -56,6 +56,17 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
+  cluster_autoscaling {
+    enabled = var.cluster_autoscaling.enabled
+    dynamic "resource_limits" {
+      for_each = local.autoscalling_resource_limits
+      content {
+        resource_type = lookup(resource_limits.value, "resource_type")
+        minimum       = lookup(resource_limits.value, "minimum")
+        maximum       = lookup(resource_limits.value, "maximum")
+      }
+    }
+  }
 
   default_max_pods_per_node = var.default_max_pods_per_node
 
@@ -266,6 +277,8 @@ resource "google_container_node_pool" "pools" {
   name     = { for k, v in random_id.name : k => v.hex }[each.key]
   project  = var.project_id
   location = local.location
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null
 
   cluster = google_container_cluster.primary.name
 

diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf
@@ -48,6 +48,16 @@ locals {
 
   release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
 
+  autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{
+    resource_type = "cpu"
+    minimum       = var.cluster_autoscaling.min_cpu_cores
+    maximum       = var.cluster_autoscaling.max_cpu_cores
+    }, {
+    resource_type = "memory"
+    minimum       = var.cluster_autoscaling.min_memory_gb
+    maximum       = var.cluster_autoscaling.max_memory_gb
+  }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0

diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf
@@ -188,6 +188,24 @@ variable "enable_resource_consumption_export" {
   default     = true
 }
 
+variable "cluster_autoscaling" {
+  type = object({
+    enabled       = bool
+    min_cpu_cores = number
+    max_cpu_cores = number
+    min_memory_gb = number
+    max_memory_gb = number
+  })
+  default = {
+    enabled       = false
+    max_cpu_cores = 0
+    min_cpu_cores = 0
+    max_memory_gb = 0
+    min_memory_gb = 0
+  }
+  description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)"
+}
+
 variable "node_pools_taints" {
   type        = map(list(object({ key = string, value = string, effect = string })))
   description = "Map of lists containing node taints by node-pool name"

diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md
@@ -109,6 +109,7 @@ Then perform the following commands on the root folder:
 | add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
+| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `<map>` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |

diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf
@@ -56,6 +56,17 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
+  cluster_autoscaling {
+    enabled = var.cluster_autoscaling.enabled
+    dynamic "resource_limits" {
+      for_each = local.autoscalling_resource_limits
+      content {
+        resource_type = lookup(resource_limits.value, "resource_type")
+        minimum       = lookup(resource_limits.value, "minimum")
+        maximum       = lookup(resource_limits.value, "maximum")
+      }
+    }
+  }
 
   default_max_pods_per_node = var.default_max_pods_per_node
 
@@ -194,6 +205,8 @@ resource "google_container_node_pool" "pools" {
   name     = each.key
   project  = var.project_id
   location = local.location
+  // use node_locations if provided, defaults to cluster level node_locations if not specified
+  node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null
 
   cluster = google_container_cluster.primary.name
 

diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf
@@ -48,6 +48,16 @@ locals {
 
   release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
 
+  autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{
+    resource_type = "cpu"
+    minimum       = var.cluster_autoscaling.min_cpu_cores
+    maximum       = var.cluster_autoscaling.max_cpu_cores
+    }, {
+    resource_type = "memory"
+    minimum       = var.cluster_autoscaling.min_memory_gb
+    maximum       = var.cluster_autoscaling.max_memory_gb
+  }] : []
+
 
   custom_kube_dns_config      = length(keys(var.stub_domains)) > 0
   upstream_nameservers_config = length(var.upstream_nameservers) > 0

diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf
@@ -188,6 +188,24 @@ variable "enable_resource_consumption_export" {
   default     = true
 }
 
+variable "cluster_autoscaling" {
+  type = object({
+    enabled       = bool
+    min_cpu_cores = number
+    max_cpu_cores = number
+    min_memory_gb = number
+    max_memory_gb = number
+  })
+  default = {
+    enabled       = false
+    max_cpu_cores = 0
+    min_cpu_cores = 0
+    max_memory_gb = 0
+    min_memory_gb = 0
+  }
+  description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)"
+}
+
 variable "node_pools_taints" {
   type        = map(list(object({ key = string, value = string, effect = string })))
   description = "Map of lists containing node taints by node-pool name"

diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb
@@ -40,7 +40,7 @@
                 "oauthScopes" => %w(https://www.googleapis.com/auth/logging.write https://www.googleapis.com/auth/monitoring),
                 "serviceAccount" => "default"
             },
-            "autoscalingProfile" => "OPTIMIZE_UTILIZATION",
+            "autoscalingProfile" => "BALANCED",
             "enableNodeAutoprovisioning" => true,
             "resourceLimits" => [
                 {

diff --git a/variables.tf b/variables.tf
@@ -188,6 +188,24 @@ variable "enable_resource_consumption_export" {
   default     = true
 }
 
+variable "cluster_autoscaling" {
+  type = object({
+    enabled       = bool
+    min_cpu_cores = number
+    max_cpu_cores = number
+    min_memory_gb = number
+    max_memory_gb = number
+  })
+  default = {
+    enabled       = false
+    max_cpu_cores = 0
+    min_cpu_cores = 0
+    max_memory_gb = 0
+    min_memory_gb = 0
+  }
+  description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)"
+}
+
 variable "node_pools_taints" {
   type        = map(list(object({ key = string, value = string, effect = string })))
   description = "Map of lists containing node taints by node-pool name"