feat: Rollout default_max_pods_per_node setting to GA modules (#439)

* roll out max_pods_per_node

* fix outputs

* add test for max_pods_per_node

README.md

@@ -118,6 +118,7 @@ Then perform the following commands on the root folder:
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |

autogen/main/cluster.tf.tmpl

@@ -78,10 +78,12 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+{% endif %}
 
+  default_max_pods_per_node = var.default_max_pods_per_node
+{% if beta_cluster %}
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
-  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_shielded_nodes       = var.enable_shielded_nodes
   enable_kubernetes_alpha     = var.enable_kubernetes_alpha
 
@@ -369,9 +371,7 @@ resource "google_container_node_pool" "pools" {
     lookup(each.value, "min_count", 1)
   ) : null
 
-  {% if beta_cluster %}
   max_pods_per_node = lookup(each.value, "max_pods_per_node", null)
-  {% endif %}
 
   node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1)
 

autogen/main/variables.tf.tmpl

@@ -353,6 +353,11 @@ variable "skip_provisioners" {
   description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }
+
+variable "default_max_pods_per_node" {
+  description = "The maximum number of pods to schedule per node"
+  default     = 110
+}
 {% if private_cluster %}
 
 variable "deploy_using_private_endpoint" {
@@ -386,11 +391,6 @@ variable "istio" {
   default     = false
 }
 
-variable "default_max_pods_per_node" {
-  description = "The maximum number of pods to schedule per node"
-  default     = 110
-}
-
 variable "database_encryption" {
   description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key."
   type        = list(object({ state = string, key_name = string }))

cluster.tf

@@ -49,6 +49,8 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
+
+  default_max_pods_per_node = var.default_max_pods_per_node
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -143,6 +145,7 @@ resource "google_container_node_pool" "pools" {
     lookup(each.value, "min_count", 1)
   ) : null
 
+  max_pods_per_node = lookup(each.value, "max_pods_per_node", null)
 
   node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1)
 

examples/simple_regional_private/main.tf

@@ -30,20 +30,40 @@ data "google_compute_subnetwork" "subnetwork" {
 }
 
 module "gke" {
-  source                  = "../../modules/private-cluster/"
-  project_id              = var.project_id
-  name                    = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
-  regional                = true
-  region                  = var.region
-  network                 = var.network
-  subnetwork              = var.subnetwork
-  ip_range_pods           = var.ip_range_pods
-  ip_range_services       = var.ip_range_services
-  create_service_account  = false
-  service_account         = var.compute_engine_service_account
-  enable_private_endpoint = true
-  enable_private_nodes    = true
-  master_ipv4_cidr_block  = "172.16.0.0/28"
+  source                    = "../../modules/private-cluster/"
+  project_id                = var.project_id
+  name                      = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional                  = true
+  region                    = var.region
+  network                   = var.network
+  subnetwork                = var.subnetwork
+  ip_range_pods             = var.ip_range_pods
+  ip_range_services         = var.ip_range_services
+  create_service_account    = false
+  service_account           = var.compute_engine_service_account
+  enable_private_endpoint   = true
+  enable_private_nodes      = true
+  master_ipv4_cidr_block    = "172.16.0.0/28"
+  default_max_pods_per_node = 20
+  remove_default_node_pool  = true
+
+  node_pools = [
+    {
+      name              = "pool-01"
+      machine_type      = "n1-standard-2"
+      min_count         = 1
+      max_count         = 100
+      local_ssd_count   = 0
+      disk_size_gb      = 100
+      disk_type         = "pd-standard"
+      image_type        = "COS"
+      auto_repair       = true
+      auto_upgrade      = true
+      service_account   = var.compute_engine_service_account
+      preemptible       = false
+      max_pods_per_node = 12
+    },
+  ]
 
   master_authorized_networks = [
     {

modules/beta-private-cluster-update-variant/cluster.tf

@@ -68,9 +68,9 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
-  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_shielded_nodes       = var.enable_shielded_nodes
   enable_kubernetes_alpha     = var.enable_kubernetes_alpha
 

modules/beta-private-cluster-update-variant/variables.tf

@@ -350,6 +350,11 @@ variable "skip_provisioners" {
   default     = false
 }
 
+variable "default_max_pods_per_node" {
+  description = "The maximum number of pods to schedule per node"
+  default     = 110
+}
+
 variable "deploy_using_private_endpoint" {
   type        = bool
   description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment."
@@ -379,11 +384,6 @@ variable "istio" {
   default     = false
 }
 
-variable "default_max_pods_per_node" {
-  description = "The maximum number of pods to schedule per node"
-  default     = 110
-}
-
 variable "database_encryption" {
   description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key."
   type        = list(object({ state = string, key_name = string }))

modules/beta-private-cluster/cluster.tf

@@ -68,9 +68,9 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
-  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_shielded_nodes       = var.enable_shielded_nodes
   enable_kubernetes_alpha     = var.enable_kubernetes_alpha
 

modules/beta-private-cluster/variables.tf

@@ -350,6 +350,11 @@ variable "skip_provisioners" {
   default     = false
 }
 
+variable "default_max_pods_per_node" {
+  description = "The maximum number of pods to schedule per node"
+  default     = 110
+}
+
 variable "deploy_using_private_endpoint" {
   type        = bool
   description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment."
@@ -379,11 +384,6 @@ variable "istio" {
   default     = false
 }
 
-variable "default_max_pods_per_node" {
-  description = "The maximum number of pods to schedule per node"
-  default     = 110
-}
-
 variable "database_encryption" {
   description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key."
   type        = list(object({ state = string, key_name = string }))

modules/beta-public-cluster/cluster.tf

@@ -68,9 +68,9 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
-  default_max_pods_per_node   = var.default_max_pods_per_node
   enable_shielded_nodes       = var.enable_shielded_nodes
   enable_kubernetes_alpha     = var.enable_kubernetes_alpha
 

modules/beta-public-cluster/variables.tf

@@ -350,16 +350,16 @@ variable "skip_provisioners" {
   default     = false
 }
 
-variable "istio" {
-  description = "(Beta) Enable Istio addon"
-  default     = false
-}
-
 variable "default_max_pods_per_node" {
   description = "The maximum number of pods to schedule per node"
   default     = 110
 }
 
+variable "istio" {
+  description = "(Beta) Enable Istio addon"
+  default     = false
+}
+
 variable "database_encryption" {
   description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key."
   type        = list(object({ state = string, key_name = string }))

modules/private-cluster-update-variant/README.md

@@ -158,6 +158,7 @@ Then perform the following commands on the root folder:
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |

modules/private-cluster-update-variant/cluster.tf

@@ -49,6 +49,8 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
+
+  default_max_pods_per_node = var.default_max_pods_per_node
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -228,6 +230,7 @@ resource "google_container_node_pool" "pools" {
     lookup(each.value, "min_count", 1)
   ) : null
 
+  max_pods_per_node = lookup(each.value, "max_pods_per_node", null)
 
   node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1)
 

modules/private-cluster-update-variant/variables.tf

@@ -303,6 +303,11 @@ variable "skip_provisioners" {
   default     = false
 }
 
+variable "default_max_pods_per_node" {
+  description = "The maximum number of pods to schedule per node"
+  default     = 110
+}
+
 variable "deploy_using_private_endpoint" {
   type        = bool
   description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment."

modules/private-cluster/README.md

@@ -136,6 +136,7 @@ Then perform the following commands on the root folder:
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `<map>` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no |
 | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no |
+| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |

modules/private-cluster/cluster.tf

@@ -49,6 +49,8 @@ resource "google_container_cluster" "primary" {
   logging_service    = var.logging_service
   monitoring_service = var.monitoring_service
 
+
+  default_max_pods_per_node = var.default_max_pods_per_node
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -156,6 +158,7 @@ resource "google_container_node_pool" "pools" {
     lookup(each.value, "min_count", 1)
   ) : null
 
+  max_pods_per_node = lookup(each.value, "max_pods_per_node", null)
 
   node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1)
 

modules/private-cluster/variables.tf

@@ -303,6 +303,11 @@ variable "skip_provisioners" {
   default     = false
 }
 
+variable "default_max_pods_per_node" {
+  description = "The maximum number of pods to schedule per node"
+  default     = 110
+}
+
 variable "deploy_using_private_endpoint" {
   type        = bool
   description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment."

test/integration/simple_regional_private/controls/gcloud.rb

@@ -47,6 +47,10 @@
         expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true
       end
 
+      it "has 20 max pods" do
+        expect(data['defaultMaxPodsConstraint']['maxPodsPerNode']).to eq "20"
+      end
+
       it "has the expected addon settings" do
         expect(data['addonsConfig']).to eq({
           "horizontalPodAutoscaling" => {},
@@ -63,7 +67,7 @@
       it "exists" do
         expect(data['nodePools']).to include(
           including(
-            "name" => "default-pool",
+            "name" => "pool-01",
           )
         )
       end
@@ -128,7 +132,7 @@
             "config" => including(
               "labels" => including(
                 "cluster_name" => cluster_name,
-                "node_pool" => "default-node-pool",
+                "node_pool" => "pool-01",
               ),
             ),
           )
@@ -141,7 +145,7 @@
             "config" => including(
               "tags" => match_array([
                 "gke-#{cluster_name}",
-                "gke-#{cluster_name}-default-node-pool",
+                "gke-#{cluster_name}-pool-01",
               ]),
             ),
           )
@@ -157,6 +161,16 @@
           )
         )
       end
+
+      it "has 12 max pods" do
+        expect(node_pools).to include(
+          including(
+            "maxPodsConstraint" => including(
+              "maxPodsPerNode" => "12",
+            ),
+          )
+        )
+      end
     end
   end
 end

variables.tf

@@ -302,3 +302,8 @@ variable "skip_provisioners" {
   description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality."
   default     = false
 }
+
+variable "default_max_pods_per_node" {
+  description = "The maximum number of pods to schedule per node"
+  default     = 110
+}