From 36ddbbb82d6861689d734e76eeab7c0d162351ce Mon Sep 17 00:00:00 2001 From: Anshu Rao Date: Sun, 23 Feb 2020 09:13:53 -0600 Subject: [PATCH] feat: Rollout default_max_pods_per_node setting to GA modules (#439) * roll out max_pods_per_node * fix outputs * add test for max_pods_per_node --- README.md | 1 + autogen/main/cluster.tf.tmpl | 6 +-- autogen/main/variables.tf.tmpl | 10 ++-- cluster.tf | 3 ++ examples/simple_regional_private/main.tf | 48 +++++++++++++------ .../cluster.tf | 2 +- .../variables.tf | 10 ++-- modules/beta-private-cluster/cluster.tf | 2 +- modules/beta-private-cluster/variables.tf | 10 ++-- modules/beta-public-cluster/cluster.tf | 2 +- modules/beta-public-cluster/variables.tf | 10 ++-- .../private-cluster-update-variant/README.md | 1 + .../private-cluster-update-variant/cluster.tf | 3 ++ .../variables.tf | 5 ++ modules/private-cluster/README.md | 1 + modules/private-cluster/cluster.tf | 3 ++ modules/private-cluster/variables.tf | 5 ++ .../controls/gcloud.rb | 20 ++++++-- variables.tf | 5 ++ 19 files changed, 104 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 74644fdab..61b664aca 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Then perform the following commands on the root folder: | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 56bae0764..b2b4245a0 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -78,10 +78,12 @@ resource "google_container_cluster" "primary" { } } } +{% endif %} + default_max_pods_per_node = var.default_max_pods_per_node +{% if beta_cluster %} enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha @@ -369,9 +371,7 @@ resource "google_container_node_pool" "pools" { lookup(each.value, "min_count", 1) ) : null - {% if beta_cluster %} max_pods_per_node = lookup(each.value, "max_pods_per_node", null) - {% endif %} node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1) diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 54cc6b3f2..bedce28b5 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -353,6 +353,11 @@ variable "skip_provisioners" { description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." default = false } + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} {% if private_cluster %} variable "deploy_using_private_endpoint" { @@ -386,11 +391,6 @@ variable "istio" { default = false } -variable "default_max_pods_per_node" { - description = "The maximum number of pods to schedule per node" - default = 110 -} - variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/cluster.tf b/cluster.tf index 4d484c7ba..439827914 100644 --- a/cluster.tf +++ b/cluster.tf @@ -49,6 +49,8 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + + default_max_pods_per_node = var.default_max_pods_per_node dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -143,6 +145,7 @@ resource "google_container_node_pool" "pools" { lookup(each.value, "min_count", 1) ) : null + max_pods_per_node = lookup(each.value, "max_pods_per_node", null) node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1) diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index d61a202fc..8b1a837df 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -30,20 +30,40 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/private-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account - enable_private_endpoint = true - enable_private_nodes = true - master_ipv4_cidr_block = "172.16.0.0/28" + source = "../../modules/private-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + default_max_pods_per_node = 20 + remove_default_node_pool = true + + node_pools = [ + { + name = "pool-01" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS" + auto_repair = true + auto_upgrade = true + service_account = var.compute_engine_service_account + preemptible = false + max_pods_per_node = 12 + }, + ] master_authorized_networks = [ { diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 67039cd1e..b870fd244 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -68,9 +68,9 @@ resource "google_container_cluster" "primary" { } } + default_max_pods_per_node = var.default_max_pods_per_node enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index b1de22138..f320bb248 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -350,6 +350,11 @@ variable "skip_provisioners" { default = false } +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + variable "deploy_using_private_endpoint" { type = bool description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." @@ -379,11 +384,6 @@ variable "istio" { default = false } -variable "default_max_pods_per_node" { - description = "The maximum number of pods to schedule per node" - default = 110 -} - variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index afbb37c9a..1c73331c0 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -68,9 +68,9 @@ resource "google_container_cluster" "primary" { } } + default_max_pods_per_node = var.default_max_pods_per_node enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index b1de22138..f320bb248 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -350,6 +350,11 @@ variable "skip_provisioners" { default = false } +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + variable "deploy_using_private_endpoint" { type = bool description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." @@ -379,11 +384,6 @@ variable "istio" { default = false } -variable "default_max_pods_per_node" { - description = "The maximum number of pods to schedule per node" - default = 110 -} - variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index a54d075e5..3fc4c7503 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -68,9 +68,9 @@ resource "google_container_cluster" "primary" { } } + default_max_pods_per_node = var.default_max_pods_per_node enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 54fe1dd43..4732035bd 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -350,16 +350,16 @@ variable "skip_provisioners" { default = false } -variable "istio" { - description = "(Beta) Enable Istio addon" - default = false -} - variable "default_max_pods_per_node" { description = "The maximum number of pods to schedule per node" default = 110 } +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index e9643423d..27b964127 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -158,6 +158,7 @@ Then perform the following commands on the root folder: | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 955cd6389..04b3a7892 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -49,6 +49,8 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + + default_max_pods_per_node = var.default_max_pods_per_node dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -228,6 +230,7 @@ resource "google_container_node_pool" "pools" { lookup(each.value, "min_count", 1) ) : null + max_pods_per_node = lookup(each.value, "max_pods_per_node", null) node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1) diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index e90f542fd..a5986705e 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -303,6 +303,11 @@ variable "skip_provisioners" { default = false } +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + variable "deploy_using_private_endpoint" { type = bool description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 6068ca27a..c341638dd 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -136,6 +136,7 @@ Then perform the following commands on the root folder: | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 9587f8bad..d53b5aca8 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -49,6 +49,8 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + + default_max_pods_per_node = var.default_max_pods_per_node dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -156,6 +158,7 @@ resource "google_container_node_pool" "pools" { lookup(each.value, "min_count", 1) ) : null + max_pods_per_node = lookup(each.value, "max_pods_per_node", null) node_count = lookup(each.value, "autoscaling", true) ? null : lookup(each.value, "node_count", 1) diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index e90f542fd..a5986705e 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -303,6 +303,11 @@ variable "skip_provisioners" { default = false } +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + variable "deploy_using_private_endpoint" { type = bool description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." diff --git a/test/integration/simple_regional_private/controls/gcloud.rb b/test/integration/simple_regional_private/controls/gcloud.rb index b86834a2a..f5493e3eb 100644 --- a/test/integration/simple_regional_private/controls/gcloud.rb +++ b/test/integration/simple_regional_private/controls/gcloud.rb @@ -47,6 +47,10 @@ expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true end + it "has 20 max pods" do + expect(data['defaultMaxPodsConstraint']['maxPodsPerNode']).to eq "20" + end + it "has the expected addon settings" do expect(data['addonsConfig']).to eq({ "horizontalPodAutoscaling" => {}, @@ -63,7 +67,7 @@ it "exists" do expect(data['nodePools']).to include( including( - "name" => "default-pool", + "name" => "pool-01", ) ) end @@ -128,7 +132,7 @@ "config" => including( "labels" => including( "cluster_name" => cluster_name, - "node_pool" => "default-node-pool", + "node_pool" => "pool-01", ), ), ) @@ -141,7 +145,7 @@ "config" => including( "tags" => match_array([ "gke-#{cluster_name}", - "gke-#{cluster_name}-default-node-pool", + "gke-#{cluster_name}-pool-01", ]), ), ) @@ -157,6 +161,16 @@ ) ) end + + it "has 12 max pods" do + expect(node_pools).to include( + including( + "maxPodsConstraint" => including( + "maxPodsPerNode" => "12", + ), + ) + ) + end end end end diff --git a/variables.tf b/variables.tf index d0344e85a..1de98c0ac 100644 --- a/variables.tf +++ b/variables.tf @@ -302,3 +302,8 @@ variable "skip_provisioners" { description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." default = false } + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +}