Merge branch 'master' into fix/global-master-access

README.md

@@ -228,6 +228,7 @@ Then perform the following commands on the root folder:
 | release\_channel | The release channel of this cluster |
 | service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
 | type | Cluster type (regional / zonal) |
+| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled |
 | zones | List of zones in which the cluster resides |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

autogen/main/cluster.tf.tmpl

@@ -720,17 +720,14 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_oauth_scopes[each.value["name"]],
     )
 
-    guest_accelerator = [
-      for guest_accelerator in lookup(each.value, "accelerator_count", 0) > 0 ? [{
+    dynamic "guest_accelerator" {
+      for_each = lookup(each.value, "accelerator_count", 0) > 0 ? [1] : []
+      content {
         type  = lookup(each.value, "accelerator_type", "")
         count = lookup(each.value, "accelerator_count", 0)
-	gpu_partition_size = lookup(each.value, "gpu_partition_size", null)
-        }] : [] : {
-        type  = guest_accelerator["type"]
-        count = guest_accelerator["count"]
-	gpu_partition_size = guest_accelerator["gpu_partition_size"]
+        gpu_partition_size = lookup(each.value, "gpu_partition_size", null)
       }
-    ]
+    }
 
     dynamic "workload_metadata_config" {
       for_each = local.cluster_node_metadata_config

autogen/main/main.tf.tmpl

@@ -149,13 +149,13 @@ locals {
   cluster_output_network_policy_enabled             = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled
   cluster_output_http_load_balancing_enabled        = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled
   cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled
+  cluster_output_vertical_pod_autoscaling_enabled   = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false
 
 {% if beta_cluster %}
   # BETA features
   cluster_output_istio_disabled                   = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false
   cluster_output_pod_security_policy_enabled      = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false
   cluster_output_intranode_visbility_enabled      = google_container_cluster.primary.enable_intranode_visibility
-  cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false
   cluster_output_identity_service_enabled         = google_container_cluster.primary.identity_service_config != null && length(google_container_cluster.primary.identity_service_config) == 1 ? google_container_cluster.primary.identity_service_config.0.enabled : false
 
   # /BETA features
@@ -206,6 +206,7 @@ locals {
   {% endif %}
   cluster_http_load_balancing_enabled        = ! local.cluster_output_http_load_balancing_enabled
   cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled
+  cluster_vertical_pod_autoscaling_enabled   = local.cluster_output_vertical_pod_autoscaling_enabled
   workload_identity_enabled                  = ! (var.identity_namespace == null || var.identity_namespace == "null")
   cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{
     workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace
@@ -217,7 +218,6 @@ locals {
   cluster_telemetry_type_is_set            = var.cluster_telemetry_type != null
   cluster_pod_security_policy_enabled      = local.cluster_output_pod_security_policy_enabled
   cluster_intranode_visibility_enabled     = local.cluster_output_intranode_visbility_enabled
-  cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled
   confidential_node_config                 = var.enable_confidential_nodes == true ? [{ enabled = true }] : []
 
   # /BETA features

autogen/main/outputs.tf.tmpl

@@ -124,6 +124,11 @@ output "horizontal_pod_autoscaling_enabled" {
   value       = local.cluster_horizontal_pod_autoscaling_enabled
 }
 
+output "vertical_pod_autoscaling_enabled" {
+  description = "Whether vertical pod autoscaling enabled"
+  value       = local.cluster_vertical_pod_autoscaling_enabled
+}
+
 {% if autopilot_cluster != true %}
 output "node_pools_names" {
   description = "List of node pools names"
@@ -203,11 +208,6 @@ output "intranode_visibility_enabled" {
   value       = local.cluster_intranode_visibility_enabled
 }
 
-output "vertical_pod_autoscaling_enabled" {
-  description = "Whether veritical pod autoscaling is enabled"
-  value       = local.cluster_vertical_pod_autoscaling_enabled
-}
-
 output "identity_service_enabled" {
   description = "Whether Identity Service is enabled"
   value       = local.cluster_pod_security_policy_enabled

build/int.cloudbuild.yaml

@@ -131,21 +131,22 @@ steps:
     - verify simple-regional-with-networking-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-networking-local']
-- id: converge simple-zonal-local
-  waitFor:
-    - create all
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local']
-- id: verify simple-zonal-local
-  waitFor:
-    - converge simple-zonal-local
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-local']
-- id: destroy simple-zonal-local
-  waitFor:
-    - verify simple-zonal-local
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-local']
+# TODO(bharathkkb): https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1431
+# - id: converge simple-zonal-local
+#   waitFor:
+#     - create all
+#   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+#   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local']
+# - id: verify simple-zonal-local
+#   waitFor:
+#     - converge simple-zonal-local
+#   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+#   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-local']
+# - id: destroy simple-zonal-local
+#   waitFor:
+#     - verify simple-zonal-local
+#   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+#   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-local']
 - id: converge simple-zonal-private-local
   waitFor:
     - create all

cluster.tf

@@ -407,17 +407,14 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_oauth_scopes[each.value["name"]],
     )
 
-    guest_accelerator = [
-      for guest_accelerator in lookup(each.value, "accelerator_count", 0) > 0 ? [{
+    dynamic "guest_accelerator" {
+      for_each = lookup(each.value, "accelerator_count", 0) > 0 ? [1] : []
+      content {
         type               = lookup(each.value, "accelerator_type", "")
         count              = lookup(each.value, "accelerator_count", 0)
         gpu_partition_size = lookup(each.value, "gpu_partition_size", null)
-        }] : [] : {
-        type               = guest_accelerator["type"]
-        count              = guest_accelerator["count"]
-        gpu_partition_size = guest_accelerator["gpu_partition_size"]
       }
-    ]
+    }
 
     dynamic "workload_metadata_config" {
       for_each = local.cluster_node_metadata_config
@@ -560,17 +557,14 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_oauth_scopes[each.value["name"]],
     )
 
-    guest_accelerator = [
-      for guest_accelerator in lookup(each.value, "accelerator_count", 0) > 0 ? [{
+    dynamic "guest_accelerator" {
+      for_each = lookup(each.value, "accelerator_count", 0) > 0 ? [1] : []
+      content {
         type               = lookup(each.value, "accelerator_type", "")
         count              = lookup(each.value, "accelerator_count", 0)
         gpu_partition_size = lookup(each.value, "gpu_partition_size", null)
-        }] : [] : {
-        type               = guest_accelerator["type"]
-        count              = guest_accelerator["count"]
-        gpu_partition_size = guest_accelerator["gpu_partition_size"]
       }
-    ]
+    }
 
     dynamic "workload_metadata_config" {
       for_each = local.cluster_node_metadata_config

examples/acm-terraform-blog-part1/terraform/gke.tf

@@ -16,7 +16,7 @@
 
 module "enabled_google_apis" {
   source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
+  version = "~> 14.0"
 
   project_id                  = var.project
   disable_services_on_destroy = false
@@ -31,7 +31,7 @@ module "enabled_google_apis" {
 
 module "gke" {
   source             = "terraform-google-modules/kubernetes-engine/google"
-  version            = "~> 21.2"
+  version            = "~> 23.0"
   project_id         = module.enabled_google_apis.project_id
   name               = "sfl-acm-part1"
   region             = var.region

examples/acm-terraform-blog-part2/terraform/gke.tf

@@ -16,7 +16,7 @@
 
 module "enabled_google_apis" {
   source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
+  version = "~> 14.0"
 
   project_id                  = var.project
   disable_services_on_destroy = false
@@ -31,7 +31,7 @@ module "enabled_google_apis" {
 
 module "gke" {
   source             = "terraform-google-modules/kubernetes-engine/google"
-  version            = "~> 21.2"
+  version            = "~> 23.0"
   project_id         = module.enabled_google_apis.project_id
   name               = "sfl-acm-part2"
   region             = var.region

examples/acm-terraform-blog-part3/terraform/gke.tf

@@ -16,7 +16,7 @@
 
 module "enabled_google_apis" {
   source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
+  version = "~> 14.0"
 
   project_id                  = var.project
   disable_services_on_destroy = false
@@ -33,7 +33,7 @@ module "enabled_google_apis" {
 
 module "gke" {
   source             = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster"
-  version            = "~> 21.2"
+  version            = "~> 23.0"
   project_id         = module.enabled_google_apis.project_id
   name               = "sfl-acm-part3"
   region             = var.region
@@ -48,7 +48,7 @@ module "gke" {
 
 module "wi" {
   source              = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity"
-  version             = "~> 21.2"
+  version             = "~> 23.0"
   gcp_sa_name         = "cnrmsa"
   cluster_name        = module.gke.name
   name                = "cnrm-controller-manager"

examples/node_pool/main.tf

@@ -37,7 +37,7 @@ module "gke" {
   ip_range_pods                     = var.ip_range_pods
   ip_range_services                 = var.ip_range_services
   create_service_account            = false
-  remove_default_node_pool          = true
+  remove_default_node_pool          = false
   disable_legacy_metadata_endpoints = false
   cluster_autoscaling               = var.cluster_autoscaling
 
@@ -81,7 +81,7 @@ module "gke" {
 
   node_pools_metadata = {
     pool-01 = {
-      shutdown-script = file("${path.module}/data/shutdown-script.sh")
+      shutdown-script = "kubectl --kubeconfig=/var/lib/kubelet/kubeconfig drain --force=true --ignore-daemonsets=true --delete-local-data \"$HOSTNAME\""
     }
   }
 

examples/private_zonal_with_networking/main.tf

@@ -24,7 +24,7 @@ provider "kubernetes" {
 
 module "gcp-network" {
   source  = "terraform-google-modules/network/google"
-  version = ">= 4.0.1, < 5.0.0"
+  version = ">= 4.0.1"
 
   project_id   = var.project_id
   network_name = var.network

examples/regional_private_node_pool_oauth_scopes/network.tf

@@ -16,7 +16,7 @@
 
 module "gke-network" {
   source  = "terraform-google-modules/network/google"
-  version = ">= 4.0.1, < 5.0.0"
+  version = ">= 4.0.1"
 
   project_id   = var.project_id
   network_name = "random-gke-network"

examples/safer_cluster/main.tf

@@ -73,9 +73,6 @@ module "gke" {
     },
   ]
 
-  istio    = true
-  cloudrun = true
-
   notification_config_topic = google_pubsub_topic.updates.id
 }
 

examples/safer_cluster/network.tf

@@ -16,7 +16,7 @@
 
 module "gcp-network" {
   source  = "terraform-google-modules/network/google"
-  version = ">= 4.0.1, < 5.0.0"
+  version = ">= 4.0.1"
 
   project_id   = var.project_id
   network_name = local.network_name

examples/safer_cluster_iap_bastion/apis.tf

@@ -16,7 +16,7 @@
 
 module "enabled_google_apis" {
   source  = "terraform-google-modules/project-factory/google//modules/project_services"
-  version = "~> 11.3"
+  version = "~> 14.0"
 
   project_id                  = var.project_id
   disable_services_on_destroy = false

examples/safer_cluster_iap_bastion/network.tf

@@ -17,7 +17,7 @@
 
 module "vpc" {
   source  = "terraform-google-modules/network/google"
-  version = ">= 4.0.1, < 5.0.0"
+  version = ">= 4.0.1"
 
   project_id   = module.enabled_google_apis.project_id
   network_name = var.network_name
@@ -49,7 +49,7 @@ module "vpc" {
 
 module "cloud-nat" {
   source        = "terraform-google-modules/cloud-nat/google"
-  version       = "~> 1.2"
+  version       = "~> 2.0"
   project_id    = module.enabled_google_apis.project_id
   region        = var.region
   router        = "safer-router"

examples/simple_autopilot_private/network.tf

@@ -16,7 +16,7 @@
 
 module "gcp-network" {
   source  = "terraform-google-modules/network/google"
-  version = ">= 4.0.1, < 5.0.0"
+  version = ">= 4.0.1"
 
   project_id   = var.project_id
   network_name = local.network_name

examples/simple_autopilot_public/network.tf

@@ -16,7 +16,7 @@
 
 module "gcp-network" {
   source  = "terraform-google-modules/network/google"
-  version = ">= 4.0.1, < 5.0.0"
+  version = ">= 4.0.1"
 
   project_id   = var.project_id
   network_name = local.network_name

examples/simple_regional_beta/README.md

@@ -7,7 +7,6 @@ This example illustrates how to create a simple cluster with beta features.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| cloudrun | Boolean to enable / disable CloudRun | `bool` | `true` | no |
 | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
 | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes |
 | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | <pre>[<br>  {<br>    "key_name": "",<br>    "state": "DECRYPTED"<br>  }<br>]</pre> | no |
@@ -18,7 +17,6 @@ This example illustrates how to create a simple cluster with beta features.
 | gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `false` | no |
 | ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes |
-| istio | Boolean to enable / disable Istio | `bool` | `true` | no |
 | network | The VPC network to host the cluster in | `any` | n/a | yes |
 | node\_pools | List of maps containing node pools | `list(map(string))` | <pre>[<br>  {<br>    "name": "default-node-pool"<br>  }<br>]</pre> | no |
 | project\_id | The project ID to host the cluster in | `any` | n/a | yes |

examples/simple_regional_beta/main.tf

@@ -39,8 +39,6 @@ module "gke" {
   ip_range_services             = var.ip_range_services
   create_service_account        = var.compute_engine_service_account == "create"
   service_account               = var.compute_engine_service_account
-  istio                         = var.istio
-  cloudrun                      = var.cloudrun
   dns_cache                     = var.dns_cache
   gce_pd_csi_driver             = var.gce_pd_csi_driver
   sandbox_enabled               = var.sandbox_enabled

examples/simple_regional_beta/variables.tf

@@ -47,16 +47,6 @@ variable "compute_engine_service_account" {
   description = "Service account to associate to the nodes in the cluster"
 }
 
-variable "istio" {
-  description = "Boolean to enable / disable Istio"
-  default     = true
-}
-
-variable "cloudrun" {
-  description = "Boolean to enable / disable CloudRun"
-  default     = true
-}
-
 variable "dns_cache" {
   type        = bool
   description = "(Beta) The status of the NodeLocal DNSCache addon."