From ba788191e67a97d6b8eda40d7ba6e0d71c46ff52 Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Tue, 30 Jan 2024 00:44:02 +0200 Subject: [PATCH] feat: promote tpu to ga (#1856) --- README.md | 2 ++ autogen/main/cluster.tf.tmpl | 3 +- autogen/main/firewall.tf.tmpl | 3 -- autogen/main/outputs.tf.tmpl | 10 +++--- autogen/main/variables.tf.tmpl | 3 +- cluster.tf | 2 +- firewall.tf | 35 +++++++++++++++++++ .../firewall.tf | 1 - .../beta-autopilot-private-cluster/outputs.tf | 10 +++--- .../variables.tf | 1 + .../beta-autopilot-public-cluster/firewall.tf | 1 - .../beta-autopilot-public-cluster/outputs.tf | 10 +++--- .../variables.tf | 1 + .../cluster.tf | 5 ++- .../firewall.tf | 1 - .../outputs.tf | 10 +++--- .../variables.tf | 1 + modules/beta-private-cluster/cluster.tf | 5 ++- modules/beta-private-cluster/firewall.tf | 1 - modules/beta-private-cluster/outputs.tf | 10 +++--- modules/beta-private-cluster/variables.tf | 1 + .../cluster.tf | 5 ++- .../firewall.tf | 1 - .../outputs.tf | 10 +++--- .../variables.tf | 1 + modules/beta-public-cluster/cluster.tf | 5 ++- modules/beta-public-cluster/firewall.tf | 1 - modules/beta-public-cluster/outputs.tf | 10 +++--- modules/beta-public-cluster/variables.tf | 1 + .../private-cluster-update-variant/README.md | 2 ++ .../private-cluster-update-variant/cluster.tf | 2 +- .../firewall.tf | 32 +++++++++++++++++ .../private-cluster-update-variant/outputs.tf | 5 +++ .../variables.tf | 6 ++++ modules/private-cluster/README.md | 2 ++ modules/private-cluster/cluster.tf | 2 +- modules/private-cluster/firewall.tf | 32 +++++++++++++++++ modules/private-cluster/outputs.tf | 5 +++ modules/private-cluster/variables.tf | 6 ++++ outputs.tf | 5 +++ variables.tf | 6 ++++ 41 files changed, 192 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index e6a8945f5..581cf77bc 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ Then perform the following commands on the root folder: | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | @@ -257,6 +258,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index e04d86406..bec9779ea 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -189,10 +189,9 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu {% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility - enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl index 90efd57d3..73c44d1dd 100644 --- a/autogen/main/firewall.tf.tmpl +++ b/autogen/main/firewall.tf.tmpl @@ -57,7 +57,6 @@ resource "google_compute_firewall" "intra_egress" { } -{% if beta_cluster %} /****************************************** Allow egress to the TPU IPv4 CIDR block @@ -95,8 +94,6 @@ resource "google_compute_firewall" "tpu_egress" { {% endif %} } - -{% endif %} /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 832054f9c..eb8441848 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -171,6 +171,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + {% if autopilot_cluster != true %} output "mesh_certificates_config" { description = "Mesh certificates configuration" @@ -228,9 +233,4 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} {% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 7445f9d47..682ea6530 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -600,13 +600,12 @@ variable "deletion_protection" { default = true } -{% if beta_cluster %} variable "enable_tpu" { type = bool description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } -{% endif %} + {% if autopilot_cluster != true %} variable "network_policy" { type = bool diff --git a/cluster.tf b/cluster.tf index 2228fe8bb..be4e3cbc3 100644 --- a/cluster.tf +++ b/cluster.tf @@ -146,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { diff --git a/firewall.tf b/firewall.tf index 94cec9e10..b99cecd35 100644 --- a/firewall.tf +++ b/firewall.tf @@ -55,6 +55,41 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-private-cluster/firewall.tf b/modules/beta-autopilot-private-cluster/firewall.tf index 4701c82a2..3908a6336 100644 --- a/modules/beta-autopilot-private-cluster/firewall.tf +++ b/modules/beta-autopilot-private-cluster/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index 0d955524a..8ba200ea1 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -142,6 +142,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "master_ipv4_cidr_block" { @@ -183,8 +188,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 373fa527a..2a24cc62d 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -433,6 +433,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/beta-autopilot-public-cluster/firewall.tf b/modules/beta-autopilot-public-cluster/firewall.tf index 1e61965ca..b99cecd35 100644 --- a/modules/beta-autopilot-public-cluster/firewall.tf +++ b/modules/beta-autopilot-public-cluster/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index e33c8aafa..e60b61f90 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -142,6 +142,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "cloudrun_enabled" { @@ -173,8 +178,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 2c5679d00..caf41cda8 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -403,6 +403,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 434bfedbb..9aaf83847 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf index 4701c82a2..3908a6336 100644 --- a/modules/beta-private-cluster-update-variant/firewall.tf +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 72aee4055..64465ccde 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -209,8 +214,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 5c18f1472..753d7b93b 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -578,6 +578,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 1c8e68452..cc2ccb399 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf index 4701c82a2..3908a6336 100644 --- a/modules/beta-private-cluster/firewall.tf +++ b/modules/beta-private-cluster/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 72aee4055..64465ccde 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -209,8 +214,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 5c18f1472..753d7b93b 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -578,6 +578,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index a79bf757d..fcc48cf17 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-public-cluster-update-variant/firewall.tf b/modules/beta-public-cluster-update-variant/firewall.tf index 1e61965ca..b99cecd35 100644 --- a/modules/beta-public-cluster-update-variant/firewall.tf +++ b/modules/beta-public-cluster-update-variant/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index bb7cdcc72..5d199a4ed 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -199,8 +204,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index e6f3eab0e..5863324c5 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -548,6 +548,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 57f7d04c1..dddf6c018 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -152,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf index 1e61965ca..b99cecd35 100644 --- a/modules/beta-public-cluster/firewall.tf +++ b/modules/beta-public-cluster/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index bb7cdcc72..5d199a4ed 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -199,8 +204,3 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } - -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null -} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index e6f3eab0e..5863324c5 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -548,6 +548,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 646278d5c..9407ab5fb 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -193,6 +193,7 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | @@ -292,6 +293,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 2c97d719c..91f632c95 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -146,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf index a5d89cefe..3908a6336 100644 --- a/modules/private-cluster-update-variant/firewall.tf +++ b/modules/private-cluster-update-variant/firewall.tf @@ -52,6 +52,38 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 722e3b8fd..5b6a461d7 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 483f3f3f4..3d493dc4a 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -549,6 +549,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index f6f485128..54e508086 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -171,6 +171,7 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | @@ -270,6 +271,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index b410c1795..7bacc938d 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -146,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf index a5d89cefe..3908a6336 100644 --- a/modules/private-cluster/firewall.tf +++ b/modules/private-cluster/firewall.tf @@ -52,6 +52,38 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 722e3b8fd..5b6a461d7 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 483f3f3f4..3d493dc4a 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -549,6 +549,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/outputs.tf b/outputs.tf index 04bbc957f..232d0b1f2 100644 --- a/outputs.tf +++ b/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config diff --git a/variables.tf b/variables.tf index ea0472204..0b4754363 100644 --- a/variables.tf +++ b/variables.tf @@ -519,6 +519,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon"