diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2675f6e349..d355114669 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,6 +18,7 @@ name: 'lint' on: + workflow_dispatch: pull_request: branches: - master diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f56aead563..34a5677cde 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -# Copyright 2022-2023 Google LLC +# Copyright 2022-2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ jobs: if: github.repository_owner == 'GoogleCloudPlatform' || github.repository_owner == 'terraform-google-modules' runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days' diff --git a/CHANGELOG.md b/CHANGELOG.md index eef3a8bd21..48369da056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Extending the adopted spec, each change should have a link to its corresponding pull request appended. +## [30.2.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.1.0...v30.2.0) (2024-03-08) + + +### Features + +* add cross project fleet service agent ([#1896](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1896)) ([59d36b9](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/59d36b9c2ce27e2830f174afe8dd42416b664833)) + +## [30.1.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.0.0...v30.1.0) (2024-02-26) + + +### Features + +* add direct fleet registration option ([#1878](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1878)) ([6b267bd](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/6b267bd91362cd78e06850a267a04c0fd2427b1c)) +* add optional membership_location to fleet-membership ([#1860](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1860)) ([163de39](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/163de3917b3973cab9eeca83054e718c195bff14)) + + +### Bug Fixes + +* handle missing fleet membership and extend asm timeout ([#1880](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1880)) ([22896b0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/22896b0835a4f68aed92c5330c1e9a65faa97a1c)) + +## [30.0.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v29.0.0...v30.0.0) (2024-01-31) + + +### ⚠ BREAKING CHANGES + +* **TPG>=5.9:** cluster autoscaling profile is GA ([#1839](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1839)) +* Update least privilege default service account ([#1844](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1844)) +* **TPG>=5.6:** use hub membership location for output ([#1824](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1824)) +* Revert create least privilege default service account ([#1757](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1757)) (#1827) +* **TF>=1.1:** Configure ASM management mode ([#1702](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1702)) + +### Features + +* add advanced datapath observability config option ([#1776](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1776)) ([90e9bdf](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/90e9bdfb2a348ab22d996711e477cc6a7aa27c28)) +* Add support for configuring allow_net_admin in autopilot clusters ([#1768](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1768)) ([493149d](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/493149d37aa13bbda1fc8dbcaec1b40f051c642e)) +* add support for pod_range in private cluster ([#1803](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1803)) ([9c62f1f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/9c62f1f53afacf664528f9b3187c7e5df8eea1fa)) +* dual stack (IPV4_IPV6) support ([#1818](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1818)) ([d6cb390](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d6cb39062cc6ecc2f86af463afd883d1fd780657)) +* Make confidential_nodes GA ([#1815](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1815)) ([322a5ee](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/322a5ee978c5211d33c8b4605b91f0b4804994a4)) +* promote tpu to ga ([#1856](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1856)) ([ba78819](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/ba788191e67a97d6b8eda40d7ba6e0d71c46ff52)) +* **TF>=1.1:** Configure ASM management mode ([#1702](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1702)) ([a9de2d7](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/a9de2d79d68b19dfee5b6bd8ee0c646ee621bbee)) +* **TPG>=5.6:** use hub membership location for output ([#1824](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1824)) ([13e79af](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/13e79af0f7a99b8c20099431df4a14ee145f9b03)) +* **TPG>=5.9:** cluster autoscaling profile is GA ([#1839](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1839)) ([495623e](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/495623eb38621976142d1e08edea94f6250f60aa)) +* Update least privilege default service account ([#1844](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1844)) ([c63aa4f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c63aa4fc3902aa53b2eea1f5bd0b7363383cacff)) +* workload-identity: Allow passing Google Service Account display_name and description ([#1834](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1834)) ([b387621](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/b387621c54235cf64d2c4cbc86c25a08fc6248fd)) + + +### Bug Fixes + +* Add project ID to the fleet feature membership for ASM ([#1832](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1832)) ([1835f80](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/1835f80a7005a840b981dc60e999dfd8ca099184)) +* alpha option for cluster creation ([#1796](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1796)) ([67b67f3](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/67b67f38e46306c7a4e0a1690d7fcb9b551874e1)) +* **CI:** extend wait time for ACM ([#1861](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1861)) ([3d840c0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/3d840c04ad339793fd1899d8a129af594ba0a48c)) +* Do not ignore "mesh_id" label on "google_container_cluster" resource ([#1836](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1836)) ([95641a6](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/95641a6e7b1eab8d59e6da48a7e15a5e662da203)) +* Revert create least privilege default service account ([#1757](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1757)) ([#1827](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1827)) ([0d7f638](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/0d7f63858d283ffb6c66d9f3a162ee0845db57f0)) + ## [29.0.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v28.0.0...v29.0.0) (2023-11-02) diff --git a/CODEOWNERS b/CODEOWNERS index 6d0d18bb23..d075929488 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,12 @@ # NOTE: This file is automatically generated from values at: # https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/blob/master/infra/terraform/test-org/org/locals.tf -* @terraform-google-modules/cft-admins @ericyz +* @terraform-google-modules/cft-admins @ericyz @gtsorbo + +# NOTE: GitHub CODEOWNERS locations: +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-and-branch-protection + +CODEOWNERS @terraform-google-modules/cft-admins +.github/CODEOWNERS @terraform-google-modules/cft-admins +docs/CODEOWNERS @terraform-google-modules/cft-admins + diff --git a/Makefile b/Makefile index beac484374..f92db301e4 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.18 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.19 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd DOCKER_BIN ?= docker diff --git a/README.md b/README.md index 0075dcdde3..23a04c47f5 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
    disk_size     = optional(number)
    disk_type     = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -155,16 +155,19 @@ Then perform the following commands on the root folder: | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -237,6 +240,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -256,6 +260,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | @@ -335,7 +340,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP][terraform-provider-google] v5 +- [Terraform Provider for GCP][terraform-provider-google] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/autogen/main/README.md b/autogen/main/README.md index 049165f61e..83227dc2f7 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -272,9 +272,9 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ {% if beta_cluster %} -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ {% else %} -- [Terraform Provider for GCP][terraform-provider-google] v5 +- [Terraform Provider for GCP][terraform-provider-google] v5.9+ {% endif %} #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 21a2ed4d1b..b1c1ec3860 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -70,14 +70,13 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } -{% if beta_cluster %} + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { enabled = confidential_nodes.value.enabled } } -{% endif %} subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" @@ -153,9 +152,7 @@ resource "google_container_cluster" "primary" { {% endif %} } } - {% if beta_cluster %} autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" - {% endif %} dynamic "resource_limits" { for_each = local.autoscaling_resource_limits content { @@ -192,10 +189,9 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu {% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility - enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] @@ -351,6 +347,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -581,6 +584,10 @@ resource "google_container_cluster" "primary" { } } {% endif %} + {% if beta_cluster %} + + depends_on = [google_project_iam_member.service_agent] + {% endif %} } {% if autopilot_cluster != true %} /****************************************** diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl index 90efd57d37..73c44d1ddd 100644 --- a/autogen/main/firewall.tf.tmpl +++ b/autogen/main/firewall.tf.tmpl @@ -57,7 +57,6 @@ resource "google_compute_firewall" "intra_egress" { } -{% if beta_cluster %} /****************************************** Allow egress to the TPU IPv4 CIDR block @@ -95,8 +94,6 @@ resource "google_compute_firewall" "tpu_egress" { {% endif %} } - -{% endif %} /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index d5c4bf784d..49b847fcba 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -60,6 +60,8 @@ locals { windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) {% endif %} + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -219,6 +221,7 @@ locals { cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] {% if autopilot_cluster != true %} cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates @@ -234,7 +237,6 @@ locals { {% endif %} cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features {% endif %} diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 832054f9cc..afedce7f55 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -171,6 +171,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + {% if autopilot_cluster != true %} output "mesh_certificates_config" { description = "Mesh certificates configuration" @@ -228,9 +233,9 @@ output "identity_service_enabled" { description = "Whether Identity Service is enabled" value = local.cluster_pod_security_policy_enabled } +{% endif %} -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } -{% endif %} diff --git a/autogen/main/sa.tf.tmpl b/autogen/main/sa.tf.tmpl index 1ab198a8b5..a775fbe6af 100644 --- a/autogen/main/sa.tf.tmpl +++ b/autogen/main/sa.tf.tmpl @@ -65,3 +65,19 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } +{% if beta_cluster %} + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} +{% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 060731e294..009c181324 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -245,9 +245,7 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ enabled = bool -{% if beta_cluster %} autoscaling_profile = string -{% endif %} min_cpu_cores = number max_cpu_cores = number min_memory_gb = number @@ -260,9 +258,7 @@ variable "cluster_autoscaling" { }) default = { enabled = false -{% if beta_cluster %} autoscaling_profile = "BALANCED" -{% endif %} max_cpu_cores = 0 min_cpu_cores = 0 max_memory_gb = 0 @@ -445,8 +441,12 @@ variable "enable_private_nodes" { variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "(Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + {% if autopilot_cluster == true%} + default = null + {% else %} default = "10.0.0.0/28" + {% endif %} } variable "master_global_access_enabled" { @@ -548,12 +548,12 @@ variable "shadow_firewall_rules_log_config" { } } -{% if beta_cluster %} variable "enable_confidential_nodes" { type = bool description = "An optional flag to enable confidential node config." default = false } +{% if beta_cluster %} variable "workload_vulnerability_mode" { description = "(beta) Vulnerability mode." @@ -604,13 +604,12 @@ variable "deletion_protection" { default = true } -{% if beta_cluster %} variable "enable_tpu" { type = bool description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } -{% endif %} + {% if autopilot_cluster != true %} variable "network_policy" { type = bool @@ -862,3 +861,17 @@ variable "allow_net_admin" { default = null } {% endif %} + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} +{% if beta_cluster %} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} +{% endif %} diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 9f5ccf71f7..c0668b2330 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -24,11 +24,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -40,13 +40,13 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.2.0" } {% else %} required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -58,7 +58,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.2.0" } {% endif %} } diff --git a/autogen/safer-cluster/versions.tf.tmpl b/autogen/safer-cluster/versions.tf.tmpl index 58932dde10..3135a334e1 100644 --- a/autogen/safer-cluster/versions.tf.tmpl +++ b/autogen/safer-cluster/versions.tf.tmpl @@ -23,6 +23,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.2.0" } } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 4aa54827ab..dc1ebab01a 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -519,6 +519,6 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.18' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.19' options: machineType: 'N1_HIGHCPU_8' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index db1450d6a9..d2d4c0c066 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -22,7 +22,7 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.18' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.19' options: machineType: 'N1_HIGHCPU_8' env: diff --git a/cluster.tf b/cluster.tf index c3b9d7d5b9..2236446061 100644 --- a/cluster.tf +++ b/cluster.tf @@ -65,6 +65,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "confidential_nodes" { + for_each = local.confidential_node_config + content { + enabled = confidential_nodes.value.enabled + } + } + subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" default_snat_status { @@ -115,6 +122,7 @@ resource "google_container_cluster" "primary" { } } + autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" dynamic "resource_limits" { for_each = local.autoscaling_resource_limits content { @@ -138,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -223,6 +231,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services diff --git a/examples/acm-terraform-blog-part1/terraform/gke.tf b/examples/acm-terraform-blog-part1/terraform/gke.tf index 93a50350ca..9c997717dc 100644 --- a/examples/acm-terraform-blog-part1/terraform/gke.tf +++ b/examples/acm-terraform-blog-part1/terraform/gke.tf @@ -31,7 +31,7 @@ module "enabled_google_apis" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = module.enabled_google_apis.project_id name = "sfl-acm-part1" region = var.region diff --git a/examples/acm-terraform-blog-part2/terraform/gke.tf b/examples/acm-terraform-blog-part2/terraform/gke.tf index 0b1d290ec5..2c0d637463 100644 --- a/examples/acm-terraform-blog-part2/terraform/gke.tf +++ b/examples/acm-terraform-blog-part2/terraform/gke.tf @@ -31,7 +31,7 @@ module "enabled_google_apis" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = module.enabled_google_apis.project_id name = "sfl-acm-part2" region = var.region diff --git a/examples/acm-terraform-blog-part3/terraform/gke.tf b/examples/acm-terraform-blog-part3/terraform/gke.tf index 1828c250f8..9f43da79bc 100644 --- a/examples/acm-terraform-blog-part3/terraform/gke.tf +++ b/examples/acm-terraform-blog-part3/terraform/gke.tf @@ -33,7 +33,7 @@ module "enabled_google_apis" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = module.enabled_google_apis.project_id name = "sfl-acm-part3" region = var.region @@ -48,7 +48,7 @@ module "gke" { module "wi" { source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = "~> 29.0" + version = "~> 30.0" gcp_sa_name = "cnrmsa" cluster_name = module.gke.name name = "cnrm-controller-manager" diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index eca6e989c3..e80252d2a9 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" @@ -55,7 +55,7 @@ resource "kubernetes_pod" "nginx-example" { spec { container { - image = "nginx:1.25.3" + image = "nginx:1.25.4" name = "nginx-example" } } diff --git a/examples/disable_client_cert/main.tf b/examples/disable_client_cert/main.tf index b44178ab04..af1e3f141d 100644 --- a/examples/disable_client_cert/main.tf +++ b/examples/disable_client_cert/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 59eb35cd19..cea54d19b6 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf index 10489bc5a9..b18de30482 100644 --- a/examples/node_pool_update_variant/main.tf +++ b/examples/node_pool_update_variant/main.tf @@ -34,7 +34,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster-update-variant" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/node_pool_update_variant_beta/main.tf b/examples/node_pool_update_variant_beta/main.tf index 438a0b0ec2..c78d2bd050 100644 --- a/examples/node_pool_update_variant_beta/main.tf +++ b/examples/node_pool_update_variant_beta/main.tf @@ -39,7 +39,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster-update-variant" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/node_pool_update_variant_public_beta/main.tf b/examples/node_pool_update_variant_public_beta/main.tf index c096e1c28c..dcab030f8b 100644 --- a/examples/node_pool_update_variant_public_beta/main.tf +++ b/examples/node_pool_update_variant_public_beta/main.tf @@ -39,7 +39,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster-update-variant" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index 889582fb8e..1f7f4c2047 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -61,7 +61,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = var.cluster_name diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf index 98c3c561dd..0bba21cdaa 100644 --- a/examples/regional_private_node_pool_oauth_scopes/main.tf +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -16,7 +16,7 @@ module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "random-test-cluster" diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 771f4970ff..ceea648aa5 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -52,7 +52,7 @@ resource "random_shuffle" "version" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/safer-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster-${random_string.suffix.result}" diff --git a/examples/safer_cluster_iap_bastion/cluster.tf b/examples/safer_cluster_iap_bastion/cluster.tf index 2525476181..9debb69001 100644 --- a/examples/safer_cluster_iap_bastion/cluster.tf +++ b/examples/safer_cluster_iap_bastion/cluster.tf @@ -16,7 +16,7 @@ module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/safer-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = module.enabled_google_apis.project_id name = var.cluster_name diff --git a/examples/safer_cluster_iap_bastion/kms.tf b/examples/safer_cluster_iap_bastion/kms.tf index 1ca442d9dc..668181500b 100644 --- a/examples/safer_cluster_iap_bastion/kms.tf +++ b/examples/safer_cluster_iap_bastion/kms.tf @@ -16,7 +16,7 @@ module "kms" { source = "terraform-google-modules/kms/google" - version = "~> 2.2.3" + version = "~> 2.3" project_id = var.project_id location = var.region keyring = "gke-keyring" diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf index d95626f694..5ff24c3c02 100644 --- a/examples/shared_vpc/main.tf +++ b/examples/shared_vpc/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_autopilot_private/main.tf b/examples/simple_autopilot_private/main.tf index 33bc104262..125ba24706 100644 --- a/examples/simple_autopilot_private/main.tf +++ b/examples/simple_autopilot_private/main.tf @@ -35,7 +35,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster" @@ -49,7 +49,6 @@ module "gke" { enable_vertical_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true - master_ipv4_cidr_block = "172.16.0.0/28" network_tags = [local.cluster_type] deletion_protection = false diff --git a/examples/simple_autopilot_private_non_default_sa/main.tf b/examples/simple_autopilot_private_non_default_sa/main.tf index 0bb1fde3e6..b9ff6e92a5 100644 --- a/examples/simple_autopilot_private_non_default_sa/main.tf +++ b/examples/simple_autopilot_private_non_default_sa/main.tf @@ -35,7 +35,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster" @@ -49,7 +49,6 @@ module "gke" { enable_vertical_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true - master_ipv4_cidr_block = "172.16.0.0/28" deletion_protection = false master_authorized_networks = [ diff --git a/examples/simple_autopilot_public/main.tf b/examples/simple_autopilot_public/main.tf index d791840c75..2fc462ca3a 100644 --- a/examples/simple_autopilot_public/main.tf +++ b/examples/simple_autopilot_public/main.tf @@ -34,7 +34,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-public-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster" diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index ad4122d5c2..11a34fa0c6 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" @@ -43,5 +43,6 @@ module "gke" { enable_cost_allocation = true enable_binary_authorization = var.enable_binary_authorization gcs_fuse_csi_driver = true + fleet_project = var.project_id deletion_protection = false } diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index 667965d3d1..cd76c5dcc6 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index d6a9b3f963..2b120d14d8 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -34,7 +34,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf index 861c4a08e0..43d2f7835f 100644 --- a/examples/simple_regional_private_beta/main.tf +++ b/examples/simple_regional_private_beta/main.tf @@ -34,7 +34,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_regional_private_with_cluster_version/main.tf b/examples/simple_regional_private_with_cluster_version/main.tf index bf511e6c3a..6d95353710 100644 --- a/examples/simple_regional_private_with_cluster_version/main.tf +++ b/examples/simple_regional_private_with_cluster_version/main.tf @@ -34,7 +34,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_regional_with_gateway_api/main.tf b/examples/simple_regional_with_gateway_api/main.tf index 60405c2181..b96367d767 100644 --- a/examples/simple_regional_with_gateway_api/main.tf +++ b/examples/simple_regional_with_gateway_api/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_regional_with_kubeconfig/main.tf b/examples/simple_regional_with_kubeconfig/main.tf index 18a6e574c9..a2fcc3c72e 100644 --- a/examples/simple_regional_with_kubeconfig/main.tf +++ b/examples/simple_regional_with_kubeconfig/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" @@ -45,7 +45,7 @@ module "gke" { module "gke_auth" { source = "terraform-google-modules/kubernetes-engine/google//modules/auth" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id location = module.gke.location diff --git a/examples/simple_regional_with_networking/main.tf b/examples/simple_regional_with_networking/main.tf index 9a45243897..28363ea059 100644 --- a/examples/simple_regional_with_networking/main.tf +++ b/examples/simple_regional_with_networking/main.tf @@ -53,7 +53,7 @@ module "gcp-network" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = var.cluster_name diff --git a/examples/simple_windows_node_pool/main.tf b/examples/simple_windows_node_pool/main.tf index 3dcbac05c4..2ec1464bd3 100644 --- a/examples/simple_windows_node_pool/main.tf +++ b/examples/simple_windows_node_pool/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id regional = false diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index c892b0d0bf..a541829254 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -34,7 +34,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf index 3b19ad6b27..cee5eda4d2 100644 --- a/examples/simple_zonal_with_acm/acm.tf +++ b/examples/simple_zonal_with_acm/acm.tf @@ -16,7 +16,7 @@ module "acm" { source = "terraform-google-modules/kubernetes-engine/google//modules/acm" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id location = module.gke.location diff --git a/examples/simple_zonal_with_acm/main.tf b/examples/simple_zonal_with_acm/main.tf index 9f66f7bab1..de06aa5d46 100644 --- a/examples/simple_zonal_with_acm/main.tf +++ b/examples/simple_zonal_with_acm/main.tf @@ -32,7 +32,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id regional = false diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf index d60873aa49..f9b368000a 100644 --- a/examples/simple_zonal_with_asm/main.tf +++ b/examples/simple_zonal_with_asm/main.tf @@ -28,7 +28,7 @@ data "google_project" "project" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "test-prefix-cluster-test-suffix" @@ -50,14 +50,14 @@ module "gke" { autoscaling = false auto_upgrade = true node_count = 3 - machine_type = "e2-standard-4" + machine_type = "e2-standard-8" }, ] } module "asm" { source = "terraform-google-modules/kubernetes-engine/google//modules/asm" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id cluster_name = module.gke.name diff --git a/examples/simple_zonal_with_hub/README.md b/examples/simple_zonal_with_hub/README.md index ce0a513ac0..e167ac323f 100644 --- a/examples/simple_zonal_with_hub/README.md +++ b/examples/simple_zonal_with_hub/README.md @@ -23,6 +23,7 @@ It incorporates the standard cluster module, the [registration module](../../mod | ca\_certificate | n/a | | client\_token | n/a | | cluster\_name | Cluster name | +| hub\_location | The location of the hub membership. | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | | kubernetes\_endpoint | n/a | diff --git a/examples/simple_zonal_with_hub/hub.tf b/examples/simple_zonal_with_hub/hub.tf index c838b44cbf..36120501b3 100644 --- a/examples/simple_zonal_with_hub/hub.tf +++ b/examples/simple_zonal_with_hub/hub.tf @@ -16,7 +16,7 @@ module "hub" { source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-membership" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id location = module.gke.location diff --git a/examples/simple_zonal_with_hub/main.tf b/examples/simple_zonal_with_hub/main.tf index 2fbbeddc78..4b4563d8d0 100644 --- a/examples/simple_zonal_with_hub/main.tf +++ b/examples/simple_zonal_with_hub/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id regional = false diff --git a/examples/simple_zonal_with_hub/outputs.tf b/examples/simple_zonal_with_hub/outputs.tf index d953d1b3db..d824225879 100644 --- a/examples/simple_zonal_with_hub/outputs.tf +++ b/examples/simple_zonal_with_hub/outputs.tf @@ -79,3 +79,8 @@ output "master_kubernetes_version" { description = "The master Kubernetes version" value = module.gke.master_version } + +output "hub_location" { + description = "The location of the hub membership." + value = module.hub.location +} diff --git a/examples/simple_zonal_with_hub_kubeconfig/hub.tf b/examples/simple_zonal_with_hub_kubeconfig/hub.tf index 5e500327b7..19e243947a 100644 --- a/examples/simple_zonal_with_hub_kubeconfig/hub.tf +++ b/examples/simple_zonal_with_hub_kubeconfig/hub.tf @@ -16,7 +16,7 @@ module "hub" { source = "terraform-google-modules/kubernetes-engine/google//modules/hub-legacy" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id location = "remote" diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf index 00a6c85eed..8607a6fc72 100644 --- a/examples/stub_domains/main.tf +++ b/examples/stub_domains/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf index 3c483acc13..5ebc7a057a 100644 --- a/examples/stub_domains_private/main.tf +++ b/examples/stub_domains_private/main.tf @@ -30,7 +30,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services diff --git a/examples/stub_domains_upstream_nameservers/main.tf b/examples/stub_domains_upstream_nameservers/main.tf index 75fabb9aaa..02639383dd 100644 --- a/examples/stub_domains_upstream_nameservers/main.tf +++ b/examples/stub_domains_upstream_nameservers/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/upstream_nameservers/main.tf b/examples/upstream_nameservers/main.tf index 7eb438547c..7349ab5cdf 100644 --- a/examples/upstream_nameservers/main.tf +++ b/examples/upstream_nameservers/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/examples/workload_identity/main.tf b/examples/workload_identity/main.tf index 84d05f073d..fe1b13bfcd 100644 --- a/examples/workload_identity/main.tf +++ b/examples/workload_identity/main.tf @@ -28,7 +28,7 @@ provider "kubernetes" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" @@ -54,7 +54,7 @@ module "gke" { # example without existing KSA module "workload_identity" { source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "iden-${module.gke.name}" @@ -74,7 +74,7 @@ resource "kubernetes_service_account" "test" { module "workload_identity_existing_ksa" { source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "existing-${module.gke.name}" @@ -93,7 +93,7 @@ resource "google_service_account" "custom" { module "workload_identity_existing_gsa" { source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = google_service_account.custom.account_id diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf index fce542bd95..856591c444 100644 --- a/examples/workload_metadata_config/main.tf +++ b/examples/workload_metadata_config/main.tf @@ -34,7 +34,7 @@ data "google_compute_subnetwork" "subnetwork" { module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" - version = "~> 29.0" + version = "~> 30.0" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" diff --git a/firewall.tf b/firewall.tf index 94cec9e103..b99cecd357 100644 --- a/firewall.tf +++ b/firewall.tf @@ -55,6 +55,41 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/main.tf b/main.tf index 8ad332d19d..4025b6063c 100644 --- a/main.tf +++ b/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -162,6 +164,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] diff --git a/modules/acm/creds.tf b/modules/acm/creds.tf index 75c332e74e..53f06aa916 100644 --- a/modules/acm/creds.tf +++ b/modules/acm/creds.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ resource "time_sleep" "wait_acm" { count = (var.create_ssh_key == true || var.ssh_auth_key != null || var.enable_policy_controller || var.enable_config_sync) ? 1 : 0 depends_on = [google_gke_hub_feature_membership.main] - create_duration = (length(var.policy_bundles) > 0) ? "600s" : "300s" + create_duration = "600s" } resource "google_service_account_iam_binding" "ksa_iam" { diff --git a/modules/acm/versions.tf b/modules/acm/versions.tf index 164143f2cf..fa8aea6590 100644 --- a/modules/acm/versions.tf +++ b/modules/acm/versions.tf @@ -19,11 +19,11 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v30.2.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v30.2.0" } required_providers { diff --git a/modules/asm/scripts/create_cpr.sh b/modules/asm/scripts/create_cpr.sh index d592e54538..9ceb332312 100755 --- a/modules/asm/scripts/create_cpr.sh +++ b/modules/asm/scripts/create_cpr.sh @@ -57,4 +57,4 @@ spec: channel: "${CHANNEL}" EOF -kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/"${REVISION_NAME}" --timeout 10m +kubectl wait -n istio-system --for=condition=Reconciled controlplanerevision/"${REVISION_NAME}" --timeout 20m diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf index 958bab4728..c97e274253 100644 --- a/modules/asm/versions.tf +++ b/modules/asm/versions.tf @@ -36,10 +36,10 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v30.2.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v30.2.0" } } diff --git a/modules/auth/versions.tf b/modules/auth/versions.tf index dff2c99edf..ae15435ef5 100644 --- a/modules/auth/versions.tf +++ b/modules/auth/versions.tf @@ -26,6 +26,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v30.2.0" } } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 3881493c2c..e7fc994480 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -98,6 +98,8 @@ Then perform the following commands on the root folder: | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -115,7 +117,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `null` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | @@ -153,6 +155,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -197,7 +200,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index 539b173783..b0af2e3623 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -56,6 +56,7 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { @@ -146,6 +147,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -260,4 +268,6 @@ resource "google_container_cluster" "primary" { topic = var.notification_config_topic } } + + depends_on = [google_project_iam_member.service_agent] } diff --git a/modules/beta-autopilot-private-cluster/firewall.tf b/modules/beta-autopilot-private-cluster/firewall.tf index 4701c82a2c..3908a63364 100644 --- a/modules/beta-autopilot-private-cluster/firewall.tf +++ b/modules/beta-autopilot-private-cluster/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index b5ce225a9c..f0d22b5840 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -49,6 +49,8 @@ locals { master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -121,13 +123,13 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_dns_cache_enabled = var.dns_cache cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index 0d955524ae..f2888c0aa1 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -142,6 +142,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "master_ipv4_cidr_block" { @@ -184,7 +189,7 @@ output "identity_service_enabled" { value = local.cluster_pod_security_policy_enabled } -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } diff --git a/modules/beta-autopilot-private-cluster/sa.tf b/modules/beta-autopilot-private-cluster/sa.tf index 6f89899bee..6c6e1f663b 100644 --- a/modules/beta-autopilot-private-cluster/sa.tf +++ b/modules/beta-autopilot-private-cluster/sa.tf @@ -65,3 +65,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 373fa527a2..faa9d3d1f7 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -283,8 +283,8 @@ variable "enable_private_nodes" { variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" - default = "10.0.0.0/28" + description = "(Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + default = null } variable "master_global_access_enabled" { @@ -433,6 +433,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) @@ -459,3 +460,15 @@ variable "allow_net_admin" { type = bool default = null } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf index ef228e7a97..41dd788f36 100644 --- a/modules/beta-autopilot-private-cluster/versions.tf +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v30.2.0" } } diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 7b93e80d9a..81f6883bbd 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -89,6 +89,8 @@ Then perform the following commands on the root folder: | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -142,6 +144,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -184,7 +187,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 2c9bf25fc0..43878c77b2 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -56,6 +56,7 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { @@ -146,6 +147,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -241,4 +249,6 @@ resource "google_container_cluster" "primary" { topic = var.notification_config_topic } } + + depends_on = [google_project_iam_member.service_agent] } diff --git a/modules/beta-autopilot-public-cluster/firewall.tf b/modules/beta-autopilot-public-cluster/firewall.tf index 1e61965ca2..b99cecd357 100644 --- a/modules/beta-autopilot-public-cluster/firewall.tf +++ b/modules/beta-autopilot-public-cluster/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index 8b204a0404..24bcb36d3d 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -49,6 +49,8 @@ locals { master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -120,13 +122,13 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled cluster_dns_cache_enabled = var.dns_cache cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index e33c8aafa6..9b26f835f8 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -142,6 +142,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "cloudrun_enabled" { @@ -174,7 +179,7 @@ output "identity_service_enabled" { value = local.cluster_pod_security_policy_enabled } -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } diff --git a/modules/beta-autopilot-public-cluster/sa.tf b/modules/beta-autopilot-public-cluster/sa.tf index 6f89899bee..6c6e1f663b 100644 --- a/modules/beta-autopilot-public-cluster/sa.tf +++ b/modules/beta-autopilot-public-cluster/sa.tf @@ -65,3 +65,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 2c5679d00b..29a3db949b 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -403,6 +403,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) @@ -429,3 +430,15 @@ variable "allow_net_admin" { type = bool default = null } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf index 1c27ab00be..a9fd481366 100644 --- a/modules/beta-autopilot-public-cluster/versions.tf +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v30.2.0" } } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 176188b9de..9fc4f48564 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -211,6 +211,8 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -237,7 +239,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | @@ -293,6 +295,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -406,7 +409,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 2e4d762f4d..07e0ee937b 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -64,6 +64,7 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { @@ -151,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] @@ -281,6 +281,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -493,6 +500,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf index 4701c82a2c..3908a63364 100644 --- a/modules/beta-private-cluster-update-variant/firewall.tf +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 3fea94230a..f7ef1e33d2 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -181,6 +183,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -191,7 +194,6 @@ locals { cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 72aee4055d..2d8e768f7c 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -210,7 +215,7 @@ output "identity_service_enabled" { value = local.cluster_pod_security_policy_enabled } -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf index 6f89899bee..6c6e1f663b 100644 --- a/modules/beta-private-cluster-update-variant/sa.tf +++ b/modules/beta-private-cluster-update-variant/sa.tf @@ -65,3 +65,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 5c18f14727..d149edca13 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -423,7 +423,7 @@ variable "enable_private_nodes" { variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "(Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -578,6 +578,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -810,3 +811,15 @@ variable "enable_gcfs" { description = "Enable image streaming on cluster level." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index a33ba28c09..420dfc7b60 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v30.2.0" } } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 63c4048513..df855450e7 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -189,6 +189,8 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -215,7 +217,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | @@ -271,6 +273,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -384,7 +387,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index f2975f2683..12807cbdc7 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -64,6 +64,7 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { @@ -151,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] @@ -281,6 +281,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -493,6 +500,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf index 4701c82a2c..3908a63364 100644 --- a/modules/beta-private-cluster/firewall.tf +++ b/modules/beta-private-cluster/firewall.tf @@ -84,7 +84,6 @@ resource "google_compute_firewall" "tpu_egress" { } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 3fea94230a..f7ef1e33d2 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -181,6 +183,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -191,7 +194,6 @@ locals { cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 72aee4055d..2d8e768f7c 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -210,7 +215,7 @@ output "identity_service_enabled" { value = local.cluster_pod_security_policy_enabled } -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index 6f89899bee..6c6e1f663b 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -65,3 +65,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 5c18f14727..d149edca13 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -423,7 +423,7 @@ variable "enable_private_nodes" { variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "(Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -578,6 +578,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -810,3 +811,15 @@ variable "enable_gcfs" { description = "Enable image streaming on cluster level." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index bf338436de..25e552d1d0 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v30.2.0" } } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index d091fcba9b..61d34f1e25 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -202,6 +202,8 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -282,6 +284,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -394,7 +397,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 5c28b74514..bfbf11d527 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -64,6 +64,7 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { @@ -151,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] @@ -281,6 +281,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -474,6 +481,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools diff --git a/modules/beta-public-cluster-update-variant/firewall.tf b/modules/beta-public-cluster-update-variant/firewall.tf index 1e61965ca2..b99cecd357 100644 --- a/modules/beta-public-cluster-update-variant/firewall.tf +++ b/modules/beta-public-cluster-update-variant/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 9264dab4b1..d92b11fadd 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -180,6 +182,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -190,7 +193,6 @@ locals { cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index bb7cdcc72c..e388fb4406 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -200,7 +205,7 @@ output "identity_service_enabled" { value = local.cluster_pod_security_policy_enabled } -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } diff --git a/modules/beta-public-cluster-update-variant/sa.tf b/modules/beta-public-cluster-update-variant/sa.tf index 6f89899bee..6c6e1f663b 100644 --- a/modules/beta-public-cluster-update-variant/sa.tf +++ b/modules/beta-public-cluster-update-variant/sa.tf @@ -65,3 +65,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index e6f3eab0e4..40023d59b2 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -548,6 +548,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -780,3 +781,15 @@ variable "enable_gcfs" { description = "Enable image streaming on cluster level." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index 905709d021..8af82e5096 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v30.2.0" } } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index c022ab2487..1019f74e73 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -180,6 +180,8 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -260,6 +262,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -372,7 +375,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 3c462677bb..da18df1fc0 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -64,6 +64,7 @@ resource "google_container_cluster" "primary" { enabled = var.enable_cost_allocation } } + dynamic "confidential_nodes" { for_each = local.confidential_node_config content { @@ -151,10 +152,9 @@ resource "google_container_cluster" "primary" { } } - enable_kubernetes_alpha = var.enable_kubernetes_alpha - - enable_intranode_visibility = var.enable_intranode_visibility + enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + enable_intranode_visibility = var.enable_intranode_visibility dynamic "pod_security_policy_config" { for_each = var.enable_pod_security_policy ? [var.enable_pod_security_policy] : [] @@ -281,6 +281,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services @@ -474,6 +481,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf index 1e61965ca2..b99cecd357 100644 --- a/modules/beta-public-cluster/firewall.tf +++ b/modules/beta-public-cluster/firewall.tf @@ -90,7 +90,6 @@ resource "google_compute_firewall" "tpu_egress" { ] } - /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9264dab4b1..d92b11fadd 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -180,6 +182,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] @@ -190,7 +193,6 @@ locals { cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled - confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] # /BETA features diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index bb7cdcc72c..e388fb4406 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -200,7 +205,7 @@ output "identity_service_enabled" { value = local.cluster_pod_security_policy_enabled } -output "tpu_ipv4_cidr_block" { - description = "The IP range in CIDR notation used for the TPUs" - value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership } diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index 6f89899bee..6c6e1f663b 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -65,3 +65,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index e6f3eab0e4..40023d59b2 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -548,6 +548,7 @@ variable "enable_tpu" { description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" default = false } + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -780,3 +781,15 @@ variable "enable_gcfs" { description = "Enable image streaming on cluster level." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index 76bae1a64b..ea131cbe79 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v30.2.0" } } diff --git a/modules/binary-authorization/versions.tf b/modules/binary-authorization/versions.tf index 983bbd032b..6bc50b416d 100644 --- a/modules/binary-authorization/versions.tf +++ b/modules/binary-authorization/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v30.2.0" } } diff --git a/modules/fleet-membership/README.md b/modules/fleet-membership/README.md index dd7e45cc1f..b0c4419136 100644 --- a/modules/fleet-membership/README.md +++ b/modules/fleet-membership/README.md @@ -29,6 +29,7 @@ To deploy this config: | enable\_fleet\_registration | Enables GKE Hub Registration when set to true | `bool` | `true` | no | | hub\_project\_id | The project in which the GKE Hub belongs. Defaults to GKE cluster project\_id. | `string` | `""` | no | | location | The location (zone or region) this cluster has been created in. | `string` | n/a | yes | +| membership\_location | Membership location for the cluster. Defaults to global. | `string` | `"global"` | no | | membership\_name | Membership name that uniquely represents the cluster being registered. Defaults to `$project_id-$location-$cluster_name`. | `string` | `""` | no | | project\_id | The project in which the GKE cluster belongs. | `string` | n/a | yes | diff --git a/modules/fleet-membership/main.tf b/modules/fleet-membership/main.tf index 92d5d83f7f..e57418caf0 100644 --- a/modules/fleet-membership/main.tf +++ b/modules/fleet-membership/main.tf @@ -18,7 +18,7 @@ locals { hub_project_id = var.hub_project_id == "" ? var.project_id : var.hub_project_id gke_hub_membership_name_complete = var.membership_name != "" ? var.membership_name : "${var.project_id}-${var.location}-${var.cluster_name}" gke_hub_membership_name = trimsuffix(substr(local.gke_hub_membership_name_complete, 0, 63), "-") - gke_hub_membership_location = regex(local.gke_hub_membership_location_re, data.google_container_cluster.primary.fleet[0].membership)[0] + gke_hub_membership_location = try(regex(local.gke_hub_membership_location_re, data.google_container_cluster.primary.fleet[0].membership)[0], null) gke_hub_membership_location_re = "//gkehub.googleapis.com/projects/[^/]*/locations/([^/]*)/memberships/[^/]*$" } diff --git a/modules/fleet-membership/membership.tf b/modules/fleet-membership/membership.tf index c5ef4cce39..796e3eb31f 100644 --- a/modules/fleet-membership/membership.tf +++ b/modules/fleet-membership/membership.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ resource "google_gke_hub_membership" "primary" { project = local.hub_project_id membership_id = local.gke_hub_membership_name + location = var.membership_location endpoint { gke_cluster { diff --git a/modules/fleet-membership/variables.tf b/modules/fleet-membership/variables.tf index eec5b4fd9b..df5c5462c9 100644 --- a/modules/fleet-membership/variables.tf +++ b/modules/fleet-membership/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,3 +46,10 @@ variable "membership_name" { type = string default = "" } + +// Defaults to global for backward compatibility. +variable "membership_location" { + description = "Membership location for the cluster. Defaults to global." + type = string + default = "global" +} diff --git a/modules/fleet-membership/versions.tf b/modules/fleet-membership/versions.tf index f4ef6d69f6..4529032fc3 100644 --- a/modules/fleet-membership/versions.tf +++ b/modules/fleet-membership/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v30.2.0" } } diff --git a/modules/hub-legacy/versions.tf b/modules/hub-legacy/versions.tf index e7c938dcb5..a771cb95e3 100644 --- a/modules/hub-legacy/versions.tf +++ b/modules/hub-legacy/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v30.2.0" } } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 4e2d0a90f0..ddd8cc56e6 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -165,7 +165,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
    disk_size     = optional(number)
    disk_type     = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -184,6 +184,7 @@ Then perform the following commands on the root folder: | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | @@ -192,10 +193,12 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -219,7 +222,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | @@ -270,6 +273,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -291,6 +295,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | @@ -370,7 +375,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP][terraform-provider-google] v5 +- [Terraform Provider for GCP][terraform-provider-google] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 65d072a852..c468560690 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -65,6 +65,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "confidential_nodes" { + for_each = local.confidential_node_config + content { + enabled = confidential_nodes.value.enabled + } + } + subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" default_snat_status { @@ -115,6 +122,7 @@ resource "google_container_cluster" "primary" { } } + autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" dynamic "resource_limits" { for_each = local.autoscaling_resource_limits content { @@ -138,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -223,6 +231,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf index a5d89cefe0..3908a63364 100644 --- a/modules/private-cluster-update-variant/firewall.tf +++ b/modules/private-cluster-update-variant/firewall.tf @@ -52,6 +52,38 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 386f42b3c0..f2a6e28ea8 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -163,6 +165,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 722e3b8fd1..e1e27f7bba 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -179,3 +184,8 @@ output "peering_name" { description = "The name of the peering between this cluster and the Google owned VPC." value = local.cluster_peering_name } + +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership +} diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 2f3de8bc6d..e0884739d7 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -239,28 +239,30 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { - enabled = false - max_cpu_cores = 0 - min_cpu_cores = 0 - max_memory_gb = 0 - min_memory_gb = 0 - gpu_resources = [] - auto_repair = true - auto_upgrade = true - disk_size = 100 - disk_type = "pd-standard" + enabled = false + autoscaling_profile = "BALANCED" + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + gpu_resources = [] + auto_repair = true + auto_upgrade = true + disk_size = 100 + disk_type = "pd-standard" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -415,7 +417,7 @@ variable "enable_private_nodes" { variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "(Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -511,6 +513,11 @@ variable "shadow_firewall_rules_log_config" { } } +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} variable "security_posture_mode" { description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." @@ -542,6 +549,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -708,3 +721,9 @@ variable "config_connector" { description = "Whether ConfigConnector is enabled for this cluster." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index 38c8a77480..b90321600d 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v30.2.0" } } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 2daf73c439..91d314167f 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -143,7 +143,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
    disk_size     = optional(number)
    disk_type     = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -162,6 +162,7 @@ Then perform the following commands on the root folder: | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | @@ -170,10 +171,12 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -197,7 +200,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | @@ -248,6 +251,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | @@ -269,6 +273,7 @@ Then perform the following commands on the root folder: | region | Cluster region | | release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | | zones | List of zones in which the cluster resides | @@ -348,7 +353,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 1.3+ -- [Terraform Provider for GCP][terraform-provider-google] v5 +- [Terraform Provider for GCP][terraform-provider-google] v5.9+ #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index a2ff5ec7eb..7bb8f67db2 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -65,6 +65,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "confidential_nodes" { + for_each = local.confidential_node_config + content { + enabled = confidential_nodes.value.enabled + } + } + subnetwork = "projects/${local.network_project_id}/regions/${local.region}/subnetworks/${var.subnetwork}" default_snat_status { @@ -115,6 +122,7 @@ resource "google_container_cluster" "primary" { } } + autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" dynamic "resource_limits" { for_each = local.autoscaling_resource_limits content { @@ -138,7 +146,7 @@ resource "google_container_cluster" "primary" { } enable_kubernetes_alpha = var.enable_kubernetes_alpha - + enable_tpu = var.enable_tpu dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -223,6 +231,13 @@ resource "google_container_cluster" "primary" { vulnerability_mode = var.security_posture_vulnerability_mode } + dynamic "fleet" { + for_each = var.fleet_project != null ? [1] : [] + content { + project = var.fleet_project + } + } + ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods services_secondary_range_name = var.ip_range_services diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf index a5d89cefe0..3908a63364 100644 --- a/modules/private-cluster/firewall.tf +++ b/modules/private-cluster/firewall.tf @@ -52,6 +52,38 @@ resource "google_compute_firewall" "intra_egress" { } +/****************************************** + Allow egress to the TPU IPv4 CIDR block + + This rule is defined separately from the + intra_egress rule above since it requires + an output from the google_container_cluster + resource. + + https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124 + *****************************************/ +resource "google_compute_firewall" "tpu_egress" { + count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0 + name = "gke-${substr(var.name, 0, min(36, length(var.name)))}-tpu-egress" + description = "Managed by terraform gke module: Allow pods to communicate with TPUs" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + +} + /****************************************** Allow GKE master to hit non 443 ports for Webhooks/Admission Controllers diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 386f42b3c0..f2a6e28ea8 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -54,6 +54,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) + fleet_membership = var.fleet_project != null ? google_container_cluster.primary.fleet[0].membership : null + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -163,6 +165,7 @@ locals { cluster_workload_identity_config = !local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ workload_pool = "${var.project_id}.svc.id.goog" }] : [{ workload_pool = var.identity_namespace }] + confidential_node_config = var.enable_confidential_nodes == true ? [{ enabled = true }] : [] cluster_mesh_certificates_config = local.workload_identity_enabled ? [{ enable_certificates = var.enable_mesh_certificates }] : [] diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 722e3b8fd1..e1e27f7bba 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -179,3 +184,8 @@ output "peering_name" { description = "The name of the peering between this cluster and the Google owned VPC." value = local.cluster_peering_name } + +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership +} diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 2f3de8bc6d..e0884739d7 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -239,28 +239,30 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { - enabled = false - max_cpu_cores = 0 - min_cpu_cores = 0 - max_memory_gb = 0 - min_memory_gb = 0 - gpu_resources = [] - auto_repair = true - auto_upgrade = true - disk_size = 100 - disk_type = "pd-standard" + enabled = false + autoscaling_profile = "BALANCED" + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + gpu_resources = [] + auto_repair = true + auto_upgrade = true + disk_size = 100 + disk_type = "pd-standard" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -415,7 +417,7 @@ variable "enable_private_nodes" { variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "(Beta) The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -511,6 +513,11 @@ variable "shadow_firewall_rules_log_config" { } } +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} variable "security_posture_mode" { description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." @@ -542,6 +549,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -708,3 +721,9 @@ variable "config_connector" { description = "Whether ConfigConnector is enabled for this cluster." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index c9ff13135a..db427f19ea 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v30.2.0" } } diff --git a/modules/safer-cluster-update-variant/versions.tf b/modules/safer-cluster-update-variant/versions.tf index 68db6eefba..5a1a57f882 100644 --- a/modules/safer-cluster-update-variant/versions.tf +++ b/modules/safer-cluster-update-variant/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v30.2.0" } } diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf index 25f8e45d8f..70b588b4de 100644 --- a/modules/safer-cluster/versions.tf +++ b/modules/safer-cluster/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v30.2.0" } } diff --git a/modules/services/versions.tf b/modules/services/versions.tf index 618b10df03..00f96eee6f 100644 --- a/modules/services/versions.tf +++ b/modules/services/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v30.2.0" } } diff --git a/modules/workload-identity/versions.tf b/modules/workload-identity/versions.tf index 3ed3d3d90a..48076ba01f 100644 --- a/modules/workload-identity/versions.tf +++ b/modules/workload-identity/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v30.2.0" } } diff --git a/outputs.tf b/outputs.tf index 04bbc957fe..6ed32741f2 100644 --- a/outputs.tf +++ b/outputs.tf @@ -161,6 +161,11 @@ output "identity_namespace" { ] } +output "tpu_ipv4_cidr_block" { + description = "The IP range in CIDR notation used for the TPUs" + value = var.enable_tpu ? google_container_cluster.primary.tpu_ipv4_cidr_block : null +} + output "mesh_certificates_config" { description = "Mesh certificates configuration" value = local.cluster_mesh_certificates_config @@ -169,3 +174,8 @@ output "mesh_certificates_config" { ] } + +output "fleet_membership" { + description = "Fleet membership (if registered)" + value = local.fleet_membership +} diff --git a/test/integration/deploy_service/deploy_service_test.go b/test/integration/deploy_service/deploy_service_test.go index 793763d03b..c87470d44b 100755 --- a/test/integration/deploy_service/deploy_service_test.go +++ b/test/integration/deploy_service/deploy_service_test.go @@ -43,7 +43,7 @@ func TestDeployService(t *testing.T) { k8sOpts := k8s.KubectlOptions{} listServices, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "svc", "terraform-example", "-o", "json") assert.NoError(err) - kubeService := utils.ParseJSONResult(t, listServices) + kubeService := testutils.ParseKubectlJSONResult(t, listServices) serviceIp := kubeService.Get("status.loadBalancer.ingress").Array()[0].Get("ip") serviceUrl := fmt.Sprintf("http://%s:8080", serviceIp) diff --git a/test/integration/go.mod b/test/integration/go.mod index b5cbfee26f..342f622d96 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -5,9 +5,10 @@ go 1.21 toolchain go1.21.5 require ( - github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.11.1 - github.com/gruntwork-io/terratest v0.46.9 - github.com/stretchr/testify v1.8.4 + github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.1 + github.com/gruntwork-io/terratest v0.46.11 + github.com/stretchr/testify v1.9.0 + github.com/tidwall/gjson v1.17.1 ) require ( @@ -17,6 +18,7 @@ require ( cloud.google.com/go/iam v1.1.2 // indirect cloud.google.com/go/storage v1.33.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/alexflint/go-filemutex v1.3.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go v1.45.5 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect @@ -70,7 +72,6 @@ require ( github.com/pquerna/otp v1.4.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect @@ -81,11 +82,11 @@ require ( github.com/zclconf/go-cty v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.17.0 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect @@ -96,7 +97,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/test/integration/go.sum b/test/integration/go.sum index f08da557ec..0549f453d9 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -187,11 +187,13 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.11.1 h1:S4Y7o5RKRC9Bk71VszCx9NeheWjdSAn5ejPuD1W6lNE= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.11.1/go.mod h1:v4TFK9TmX4mYyXL3v9wFXVN3A5vrt2LaVDBX2/OVU7Y= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.1 h1:8eKlk/DQeXPb6ITTLLWk/LdmyC9FRNMQF2ZR0sKaGEA= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.13.1/go.mod h1:jIatwk/2sLSDtnMaExpzZpQVuBbEhx+NeiP1obo/IlY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/alexflint/go-filemutex v1.3.0 h1:LgE+nTUWnQCyRKbpoceKZsPQbs84LivvgwUymZXdOcM= +github.com/alexflint/go-filemutex v1.3.0/go.mod h1:U0+VA/i30mGBlLCrFPGtTe9y6wGQfNAWPBTekHQ+c8A= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= @@ -374,8 +376,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0= github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk= -github.com/gruntwork-io/terratest v0.46.9 h1:2K0503TC8bhz1SQTlw6vX8SjexztXmCp3WvWxE6g22c= -github.com/gruntwork-io/terratest v0.46.9/go.mod h1:DVZG/s7eP1u3KOQJJfE6n7FDriMWpDvnj85XIlZMEM8= +github.com/gruntwork-io/terratest v0.46.11 h1:1Z9G18I2FNuH87Ro0YtjW4NH9ky4GDpfzE7+ivkPeB8= +github.com/gruntwork-io/terratest v0.46.11/go.mod h1:DVZG/s7eP1u3KOQJJfE6n7FDriMWpDvnj85XIlZMEM8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -485,11 +487,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -568,8 +570,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -725,8 +727,8 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1034,8 +1036,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/test/integration/simple_zonal/simple_zonal_test.go b/test/integration/simple_zonal/simple_zonal_test.go index 38b79b3672..cb5c796995 100644 --- a/test/integration/simple_zonal/simple_zonal_test.go +++ b/test/integration/simple_zonal/simple_zonal_test.go @@ -21,7 +21,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" @@ -78,11 +77,11 @@ func TestSimpleZonal(t *testing.T) { k8sOpts := k8s.KubectlOptions{} configNameSpace, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "ns", "config-management-system", "-o", "json") assert.NoError(err) - configkubeNS := utils.ParseJSONResult(t, configNameSpace) + configkubeNS := testutils.ParseKubectlJSONResult(t, configNameSpace) assert.Contains(configkubeNS.Get("metadata.name").String(), "config-management-system", "Namespace is Functional") gateKeeperNameSpace, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "ns", "gatekeeper-system", "-o", "json") assert.NoError(err) - gateKeeperkubeNS := utils.ParseJSONResult(t, gateKeeperNameSpace) + gateKeeperkubeNS := testutils.ParseKubectlJSONResult(t, gateKeeperNameSpace) assert.Contains(gateKeeperkubeNS.Get("metadata.name").String(), "gatekeeper-system", "Namespace is Functional") }) diff --git a/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go b/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go index 7e10477ac1..0d8e363b7a 100644 --- a/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go +++ b/test/integration/simple_zonal_with_asm/simple_zonal_with_asm_test.go @@ -20,7 +20,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" @@ -50,11 +49,11 @@ func TestSimpleZonalWithASM(t *testing.T) { k8sOpts := k8s.KubectlOptions{} listNameSpace, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "ns", "istio-system", "-o", "json") assert.NoError(err) - kubeNS := utils.ParseJSONResult(t, listNameSpace) + kubeNS := testutils.ParseKubectlJSONResult(t, listNameSpace) assert.Contains(kubeNS.Get("metadata.name").String(), "istio-system", "Namespace is Functional") listConfigMap, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "configmap", "asm-options", "-n", "istio-system", "-o", "json") assert.NoError(err) - kubeCM := utils.ParseJSONResult(t, listConfigMap) + kubeCM := testutils.ParseKubectlJSONResult(t, listConfigMap) assert.Contains(kubeCM.Get("metadata.name").String(), "asm-options", "Configmap is Present") }) diff --git a/test/integration/testutils/json.go b/test/integration/testutils/json.go new file mode 100644 index 0000000000..3e455232df --- /dev/null +++ b/test/integration/testutils/json.go @@ -0,0 +1,40 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutils + +import ( + "bytes" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/tidwall/gjson" +) + +var ( + KubectlTransientErrors = []string{ + "E022[23] .* the server is currently unable to handle the request", + } +) + +// Filter transient errors from kubectl output +func ParseKubectlJSONResult(t testing.TB, s string) gjson.Result { + bstring := []byte(s) + + for _, v := range KubectlTransientErrors { + bstring = bytes.Replace(bstring, []byte(v), []byte(""), -1) + } + + return utils.ParseJSONResult(t, string(bstring)) +} diff --git a/test/integration/testutils/retry.go b/test/integration/testutils/retry.go index 502822c858..c9fa684beb 100644 --- a/test/integration/testutils/retry.go +++ b/test/integration/testutils/retry.go @@ -16,6 +16,9 @@ package testutils var ( RetryableTransientErrors = map[string]string{ + // Error 409: unable to queue the operation + ".*Error 409.*unable to queue the operation": "Unable to queue operation.", + // Error code 409 for concurrent policy changes. ".*Error 409.*There were concurrent policy changes.*": "Concurrent policy changes.", diff --git a/variables.tf b/variables.tf index 5a6c4b16b9..ddfbb6d741 100644 --- a/variables.tf +++ b/variables.tf @@ -239,28 +239,30 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { - enabled = false - max_cpu_cores = 0 - min_cpu_cores = 0 - max_memory_gb = 0 - min_memory_gb = 0 - gpu_resources = [] - auto_repair = true - auto_upgrade = true - disk_size = 100 - disk_type = "pd-standard" + enabled = false + autoscaling_profile = "BALANCED" + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + gpu_resources = [] + auto_repair = true + auto_upgrade = true + disk_size = 100 + disk_type = "pd-standard" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -481,6 +483,11 @@ variable "shadow_firewall_rules_log_config" { } } +variable "enable_confidential_nodes" { + type = bool + description = "An optional flag to enable confidential node config." + default = false +} variable "security_posture_mode" { description = "Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`." @@ -512,6 +519,12 @@ variable "deletion_protection" { default = true } +variable "enable_tpu" { + type = bool + description = "Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive!" + default = false +} + variable "network_policy" { type = bool description = "Enable network policy addon" @@ -678,3 +691,9 @@ variable "config_connector" { description = "Whether ConfigConnector is enabled for this cluster." default = false } + +variable "fleet_project" { + description = "(Optional) Register the cluster with the fleet in this project." + type = string + default = null +} diff --git a/versions.tf b/versions.tf index 7f1995c251..202e0d1965 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v29.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v30.2.0" } }