diff --git a/README.md b/README.md index 0a2535ebd..ed106b14b 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ module "gke" { { name = "default-node-pool" machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 local_ssd_count = 0 @@ -77,6 +78,18 @@ module "gke" { } } + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = true + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + node_pools_tags = { all = [] @@ -103,15 +116,19 @@ Then perform the following commands on the root folder: | add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | | firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | gcloud\_skip\_download | Whether to skip downloading gcloud (assumes gcloud is already available outside the module) | bool | `"true"` | no | @@ -119,6 +136,7 @@ Then perform the following commands on the root folder: | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`) | string | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | @@ -135,6 +153,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"GKE_METADATA_SERVER"` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | @@ -146,6 +165,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | @@ -163,6 +183,7 @@ Then perform the following commands on the root folder: | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity namespace | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -174,6 +195,7 @@ Then perform the following commands on the root folder: | node\_pools\_names | List of node pools names | | node\_pools\_versions | List of node pools versions | | region | Cluster region | +| release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | @@ -194,17 +216,22 @@ The node_pools variable takes the following parameters: | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| effect | Effect for the taint | | Required | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| key | The key required for the taint | | Required | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusers | | Required | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | +| node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | | tags | The list of instance tags applied to all nodes | | Required | +| value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | @@ -224,7 +251,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v2.9 +- [Terraform Provider for GCP][terraform-provider-google] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/autogen/main/README.md b/autogen/main/README.md index 1ddaeedb4..0a1eb6a4c 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -79,9 +79,7 @@ module "gke" { { name = "default-node-pool" machine_type = "e2-medium" - {% if beta_cluster %} node_locations = "us-central1-b,us-central1-c" - {% endif %} min_count = 1 max_count = 100 local_ssd_count = 0 @@ -119,7 +117,6 @@ module "gke" { node-pool-metadata-custom-value = "my-node-pool" } } - {% if beta_cluster %} node_pools_taints = { all = [] @@ -132,7 +129,6 @@ module "gke" { }, ] } - {% endif %} node_pools_tags = { all = [] @@ -169,14 +165,10 @@ The node_pools variable takes the following parameters: | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | -{% if beta_cluster %} | effect | Effect for the taint | | Required | -{% endif %} | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | -{% if beta_cluster %} | key | The key required for the taint | | Required | -{% endif %} | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | @@ -188,19 +180,15 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusers | | Required | -{% if beta_cluster %} | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | -{% endif %} | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | {% if beta_cluster %} | sandbox_type | Sandbox to use for pods in the node pool | | Required | {% endif %} | service_account | The service account to be used by the Node VMs | " " | Optional | | tags | The list of instance tags applied to all nodes | | Required | -{% if beta_cluster %} | value | The value for the taint | | Required | -{% endif %} | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | @@ -221,9 +209,9 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 {% if beta_cluster %} -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 {% else %} -- [Terraform Provider for GCP][terraform-provider-google] v2.9 +- [Terraform Provider for GCP][terraform-provider-google] v3.41 {% endif %} ### Configure a Service Account diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 72b0f2bc2..4a14b75ba 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -45,7 +45,6 @@ resource "google_container_cluster" "primary" { } } -{% if beta_cluster %} dynamic "release_channel" { for_each = local.release_channel @@ -53,7 +52,6 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } -{% endif %} subnetwork = "projects/${local.network_project_id}/regions/${var.region}/subnetworks/${var.subnetwork}" @@ -62,19 +60,16 @@ resource "google_container_cluster" "primary" { disabled = var.disable_default_snat } {% endif %} -{% if beta_cluster %} min_master_version = var.release_channel != null ? null : local.master_version -{% else %} - min_master_version = local.master_version -{% endif %} logging_service = var.logging_service monitoring_service = var.monitoring_service -{% if beta_cluster %} cluster_autoscaling { enabled = var.cluster_autoscaling.enabled +{% if beta_cluster %} autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" +{% endif %} dynamic "resource_limits" { for_each = local.autoscalling_resource_limits content { @@ -84,14 +79,13 @@ resource "google_container_cluster" "primary" { } } } -{% endif %} default_max_pods_per_node = var.default_max_pods_per_node -{% if beta_cluster %} + enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization +{% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility - enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha vertical_pod_autoscaling { @@ -224,7 +218,6 @@ resource "google_container_cluster" "primary" { node_config { service_account = lookup(var.node_pools[0], "service_account", local.service_account) - {% if beta_cluster %} dynamic "workload_metadata_config" { for_each = local.cluster_node_metadata_config @@ -233,7 +226,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - {% endif %} } } @@ -278,7 +270,6 @@ resource "google_container_cluster" "primary" { {% endif %} remove_default_node_pool = var.remove_default_node_pool -{% if beta_cluster %} dynamic "database_encryption" { for_each = var.database_encryption @@ -297,6 +288,7 @@ resource "google_container_cluster" "primary" { } } +{% if beta_cluster %} dynamic "authenticator_groups_config" { for_each = local.cluster_authenticator_security_group content { @@ -397,10 +389,8 @@ resource "google_container_node_pool" "pools" { {% endif %} project = var.project_id location = local.location - {% if beta_cluster %} // use node_locations if provided, defaults to cluster level node_locations if not specified node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null - {% endif %} cluster = google_container_cluster.primary.name @@ -501,7 +491,6 @@ resource "google_container_node_pool" "pools" { count = guest_accelerator["count"] } ] - {% if beta_cluster %} dynamic "workload_metadata_config" { for_each = local.cluster_node_metadata_config @@ -510,7 +499,7 @@ resource "google_container_node_pool" "pools" { node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) } } - + {% if beta_cluster %} dynamic "sandbox_config" { for_each = local.cluster_sandbox_enabled diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 5d71830e3..cb23b9ce6 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -50,7 +50,6 @@ locals { node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) -{% if beta_cluster %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ @@ -63,8 +62,6 @@ locals { maximum = var.cluster_autoscaling.max_memory_gb }] : [] -{% endif %} - custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 @@ -95,10 +92,6 @@ locals { cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ - node_metadata = var.node_metadata - }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] @@ -106,6 +99,9 @@ locals { cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] {% endif %} + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -167,6 +163,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] {% if beta_cluster %} # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled @@ -176,10 +176,6 @@ locals { cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") - cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ - identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace - }] # /BETA features {% endif %} diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 3899ddfef..0c9d4be53 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -118,6 +118,19 @@ output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." value = local.service_account } + +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} {% if private_cluster %} output "master_ipv4_cidr_block" { @@ -161,17 +174,4 @@ output "vertical_pod_autoscaling_enabled" { description = "Whether veritical pod autoscaling is enabled" value = local.cluster_vertical_pod_autoscaling_enabled } - -output "release_channel" { - description = "The release channel of this cluster" - value = var.release_channel -} - -output "identity_namespace" { - description = "Workload Identity namespace" - value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null - depends_on = [ - google_container_cluster.primary - ] -} {% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 2205321bb..dc097ada6 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -207,11 +207,14 @@ variable "enable_kubernetes_alpha" { description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." default = false } +{% endif %} variable "cluster_autoscaling" { type = object({ enabled = bool +{% if beta_cluster %} autoscaling_profile = string +{% endif %} min_cpu_cores = number max_cpu_cores = number min_memory_gb = number @@ -219,7 +222,9 @@ variable "cluster_autoscaling" { }) default = { enabled = false +{% if beta_cluster %} autoscaling_profile = "BALANCED" +{% endif %} max_cpu_cores = 0 min_cpu_cores = 0 max_memory_gb = 0 @@ -227,7 +232,6 @@ variable "cluster_autoscaling" { } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } -{% endif %} variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) @@ -445,38 +449,17 @@ variable "config_connector" { default = false } -variable "database_encryption" { - description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({ state = string, key_name = string })) - - default = [{ - state = "DECRYPTED" - key_name = "" - }] -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false } -variable "enable_binary_authorization" { - description = "Enable BinAuthZ Admission controller" - default = false -} - variable "enable_pod_security_policy" { type = bool description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = false } -variable "node_metadata" { - description = "Specifies how node metadata is exposed to the workload running on the node" - default = "GKE_METADATA_SERVER" - type = string -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -495,21 +478,38 @@ variable "enable_vertical_pod_autoscaling" { default = false } -variable "identity_namespace" { - description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" - type = string - default = "enabled" -} - variable "authenticator_security_group" { type = string description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" default = null } +{% endif %} + +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} variable "release_channel" { type = string - description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." default = null } @@ -518,8 +518,11 @@ variable "enable_shielded_nodes" { description = "Enable Shielded Nodes features on all nodes in this cluster" default = true } -{% endif %} +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/cluster.tf b/cluster.tf index abcdbc0c8..acda1e1ac 100644 --- a/cluster.tf +++ b/cluster.tf @@ -41,17 +41,37 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { + for_each = local.release_channel + + content { + channel = release_channel.value.channel + } + } subnetwork = "projects/${local.network_project_id}/regions/${var.region}/subnetworks/${var.subnetwork}" - min_master_version = local.master_version + min_master_version = var.release_channel != null ? null : local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes + enable_binary_authorization = var.enable_binary_authorization dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -115,6 +135,14 @@ resource "google_container_cluster" "primary" { node_config { service_account = lookup(var.node_pools[0], "service_account", local.service_account) + + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } @@ -136,6 +164,24 @@ resource "google_container_cluster" "primary" { remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } + + dynamic "workload_identity_config" { + for_each = local.cluster_workload_identity_config + + content { + identity_namespace = workload_identity_config.value.identity_namespace + } + } + } /****************************************** @@ -147,6 +193,8 @@ resource "google_container_node_pool" "pools" { name = each.key project = var.project_id location = local.location + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null cluster = google_container_cluster.primary.name @@ -242,6 +290,14 @@ resource "google_container_node_pool" "pools" { } ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) + } + } + shielded_instance_config { enable_secure_boot = lookup(each.value, "enable_secure_boot", false) enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true) diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index 75d13dd21..ec003f9ae 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -51,7 +51,7 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/beta-private-cluster/" + source = "../../modules/private-cluster/" project_id = var.project_id name = var.cluster_name regional = false diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md index fe18fb279..778b1bb7c 100644 --- a/examples/simple_regional/README.md +++ b/examples/simple_regional/README.md @@ -9,6 +9,7 @@ This example illustrates how to create a simple cluster. |------|-------------|:----:|:-----:|:-----:| | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for services | string | n/a | yes | | network | The VPC network to host the cluster in | string | n/a | yes | diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index c4ddf21aa..7c05ceba9 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -24,18 +24,19 @@ provider "google" { } module "gke" { - source = "../../" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account - skip_provisioners = var.skip_provisioners + source = "../../" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + enable_binary_authorization = var.enable_binary_authorization + skip_provisioners = var.skip_provisioners } data "google_client_config" "default" { diff --git a/examples/simple_regional/variables.tf b/examples/simple_regional/variables.tf index ae6a86978..b60c83ed5 100644 --- a/examples/simple_regional/variables.tf +++ b/examples/simple_regional/variables.tf @@ -52,3 +52,8 @@ variable "skip_provisioners" { description = "Flag to skip local-exec provisioners" default = false } + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} diff --git a/examples/simple_zonal_with_asm/main.tf b/examples/simple_zonal_with_asm/main.tf index 1ad1f5b68..a07b312f5 100644 --- a/examples/simple_zonal_with_asm/main.tf +++ b/examples/simple_zonal_with_asm/main.tf @@ -23,12 +23,17 @@ provider "google-beta" { region = var.region } +provider "google" { + version = "~> 3.42.0" + region = var.region +} + data "google_project" "project" { project_id = var.project_id } module "gke" { - source = "../../modules/beta-public-cluster/" + source = "../../" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" regional = false diff --git a/examples/workload_identity/main.tf b/examples/workload_identity/main.tf index 9579d090a..2a1d89165 100644 --- a/examples/workload_identity/main.tf +++ b/examples/workload_identity/main.tf @@ -32,7 +32,7 @@ provider "kubernetes" { } module "gke" { - source = "../../modules/beta-public-cluster/" + source = "../../" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" region = var.region diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf index a861c2414..df979482a 100644 --- a/examples/workload_metadata_config/main.tf +++ b/examples/workload_metadata_config/main.tf @@ -30,7 +30,7 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/beta-private-cluster/" + source = "../../modules/private-cluster/" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" regional = false diff --git a/main.tf b/main.tf index a27a97c0a..a8feef217 100644 --- a/main.tf +++ b/main.tf @@ -46,6 +46,17 @@ locals { node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 @@ -67,6 +78,9 @@ locals { provider = null }] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -113,6 +127,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index d8f1bddb6..8dbbc599d 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -216,7 +216,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no | @@ -315,7 +315,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 65b4ac56e..e950f8ffb 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -74,9 +74,9 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha vertical_pod_autoscaling { @@ -461,7 +461,6 @@ resource "google_container_node_pool" "pools" { node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) } } - dynamic "sandbox_config" { for_each = local.cluster_sandbox_enabled diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 6f7648627..9e420ee95 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -59,7 +59,6 @@ locals { }] : [] - custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id @@ -84,16 +83,15 @@ locals { cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ - node_metadata = var.node_metadata - }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -148,6 +146,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun @@ -156,10 +158,6 @@ locals { cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") - cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ - identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace - }] # /BETA features cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 4f7b9196c..12084a72c 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -119,6 +119,19 @@ output "service_account" { value = local.service_account } +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} + output "master_ipv4_cidr_block" { description = "The IP range in CIDR notation used for the hosted master network" value = var.master_ipv4_cidr_block @@ -158,16 +171,3 @@ output "vertical_pod_autoscaling_enabled" { description = "Whether veritical pod autoscaling is enabled" value = local.cluster_vertical_pod_autoscaling_enabled } - -output "release_channel" { - description = "The release channel of this cluster" - value = var.release_channel -} - -output "identity_namespace" { - description = "Workload Identity namespace" - value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null - depends_on = [ - google_container_cluster.primary - ] -} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index f092bb5d7..d79cdf263 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -436,38 +436,17 @@ variable "config_connector" { default = false } -variable "database_encryption" { - description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({ state = string, key_name = string })) - - default = [{ - state = "DECRYPTED" - key_name = "" - }] -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false } -variable "enable_binary_authorization" { - description = "Enable BinAuthZ Admission controller" - default = false -} - variable "enable_pod_security_policy" { type = bool description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = false } -variable "node_metadata" { - description = "Specifies how node metadata is exposed to the workload running on the node" - default = "GKE_METADATA_SERVER" - type = string -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -486,21 +465,37 @@ variable "enable_vertical_pod_autoscaling" { default = false } -variable "identity_namespace" { - description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" - type = string - default = "enabled" -} - variable "authenticator_security_group" { type = string description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" default = null } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + variable "release_channel" { type = string - description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." default = null } @@ -510,6 +505,10 @@ variable "enable_shielded_nodes" { default = true } +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index d81a0d95d..2f823ac1b 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -194,7 +194,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no | @@ -293,7 +293,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 1a65d5922..af8d38d86 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -74,9 +74,9 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha vertical_pod_autoscaling { @@ -389,7 +389,6 @@ resource "google_container_node_pool" "pools" { node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) } } - dynamic "sandbox_config" { for_each = local.cluster_sandbox_enabled diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 6f7648627..9e420ee95 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -59,7 +59,6 @@ locals { }] : [] - custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id @@ -84,16 +83,15 @@ locals { cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ - node_metadata = var.node_metadata - }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -148,6 +146,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun @@ -156,10 +158,6 @@ locals { cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") - cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ - identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace - }] # /BETA features cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 4f7b9196c..12084a72c 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -119,6 +119,19 @@ output "service_account" { value = local.service_account } +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} + output "master_ipv4_cidr_block" { description = "The IP range in CIDR notation used for the hosted master network" value = var.master_ipv4_cidr_block @@ -158,16 +171,3 @@ output "vertical_pod_autoscaling_enabled" { description = "Whether veritical pod autoscaling is enabled" value = local.cluster_vertical_pod_autoscaling_enabled } - -output "release_channel" { - description = "The release channel of this cluster" - value = var.release_channel -} - -output "identity_namespace" { - description = "Workload Identity namespace" - value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null - depends_on = [ - google_container_cluster.primary - ] -} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index f092bb5d7..d79cdf263 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -436,38 +436,17 @@ variable "config_connector" { default = false } -variable "database_encryption" { - description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({ state = string, key_name = string })) - - default = [{ - state = "DECRYPTED" - key_name = "" - }] -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false } -variable "enable_binary_authorization" { - description = "Enable BinAuthZ Admission controller" - default = false -} - variable "enable_pod_security_policy" { type = bool description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = false } -variable "node_metadata" { - description = "Specifies how node metadata is exposed to the workload running on the node" - default = "GKE_METADATA_SERVER" - type = string -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -486,21 +465,37 @@ variable "enable_vertical_pod_autoscaling" { default = false } -variable "identity_namespace" { - description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" - type = string - default = "enabled" -} - variable "authenticator_security_group" { type = string description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" default = null } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + variable "release_channel" { type = string - description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." default = null } @@ -510,6 +505,10 @@ variable "enable_shielded_nodes" { default = true } +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 66ddf588c..245a3e5a1 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -205,7 +205,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no | @@ -302,7 +302,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 525996978..b1ea360ce 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -74,9 +74,9 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha vertical_pod_autoscaling { @@ -442,7 +442,6 @@ resource "google_container_node_pool" "pools" { node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) } } - dynamic "sandbox_config" { for_each = local.cluster_sandbox_enabled diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index edabbc45b..b9c44c48e 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -59,7 +59,6 @@ locals { }] : [] - custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id @@ -84,16 +83,15 @@ locals { cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ - node_metadata = var.node_metadata - }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -147,6 +145,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun @@ -155,10 +157,6 @@ locals { cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") - cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ - identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace - }] # /BETA features cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index 57172b134..9a8ec345e 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -119,6 +119,19 @@ output "service_account" { value = local.service_account } +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} + output "istio_enabled" { description = "Whether Istio is enabled" value = local.cluster_istio_enabled @@ -148,16 +161,3 @@ output "vertical_pod_autoscaling_enabled" { description = "Whether veritical pod autoscaling is enabled" value = local.cluster_vertical_pod_autoscaling_enabled } - -output "release_channel" { - description = "The release channel of this cluster" - value = var.release_channel -} - -output "identity_namespace" { - description = "Workload Identity namespace" - value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null - depends_on = [ - google_container_cluster.primary - ] -} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index a14d2c30b..72cd76365 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -405,38 +405,17 @@ variable "config_connector" { default = false } -variable "database_encryption" { - description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({ state = string, key_name = string })) - - default = [{ - state = "DECRYPTED" - key_name = "" - }] -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false } -variable "enable_binary_authorization" { - description = "Enable BinAuthZ Admission controller" - default = false -} - variable "enable_pod_security_policy" { type = bool description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = false } -variable "node_metadata" { - description = "Specifies how node metadata is exposed to the workload running on the node" - default = "GKE_METADATA_SERVER" - type = string -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -455,21 +434,37 @@ variable "enable_vertical_pod_autoscaling" { default = false } -variable "identity_namespace" { - description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" - type = string - default = "enabled" -} - variable "authenticator_security_group" { type = string description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" default = null } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + variable "release_channel" { type = string - description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." default = null } @@ -479,6 +474,10 @@ variable "enable_shielded_nodes" { default = true } +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 3c75ec6e7..39e92f081 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -183,7 +183,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | bool | `"false"` | no | @@ -280,7 +280,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 564948362..53139785b 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -74,9 +74,9 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility - enable_shielded_nodes = var.enable_shielded_nodes enable_kubernetes_alpha = var.enable_kubernetes_alpha vertical_pod_autoscaling { @@ -370,7 +370,6 @@ resource "google_container_node_pool" "pools" { node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) } } - dynamic "sandbox_config" { for_each = local.cluster_sandbox_enabled diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index edabbc45b..b9c44c48e 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -59,7 +59,6 @@ locals { }] : [] - custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id @@ -84,16 +83,15 @@ locals { cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ - node_metadata = var.node_metadata - }] - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group }] cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -147,6 +145,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun @@ -155,10 +157,6 @@ locals { cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") - cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ - identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace - }] # /BETA features cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 57172b134..9a8ec345e 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -119,6 +119,19 @@ output "service_account" { value = local.service_account } +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} + output "istio_enabled" { description = "Whether Istio is enabled" value = local.cluster_istio_enabled @@ -148,16 +161,3 @@ output "vertical_pod_autoscaling_enabled" { description = "Whether veritical pod autoscaling is enabled" value = local.cluster_vertical_pod_autoscaling_enabled } - -output "release_channel" { - description = "The release channel of this cluster" - value = var.release_channel -} - -output "identity_namespace" { - description = "Workload Identity namespace" - value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null - depends_on = [ - google_container_cluster.primary - ] -} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index a14d2c30b..72cd76365 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -405,38 +405,17 @@ variable "config_connector" { default = false } -variable "database_encryption" { - description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({ state = string, key_name = string })) - - default = [{ - state = "DECRYPTED" - key_name = "" - }] -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false } -variable "enable_binary_authorization" { - description = "Enable BinAuthZ Admission controller" - default = false -} - variable "enable_pod_security_policy" { type = bool description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = false } -variable "node_metadata" { - description = "Specifies how node metadata is exposed to the workload running on the node" - default = "GKE_METADATA_SERVER" - type = string -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -455,21 +434,37 @@ variable "enable_vertical_pod_autoscaling" { default = false } -variable "identity_namespace" { - description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" - type = string - default = "enabled" -} - variable "authenticator_security_group" { type = string description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" default = null } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + variable "release_channel" { type = string - description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." default = null } @@ -479,6 +474,10 @@ variable "enable_shielded_nodes" { default = true } +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index c0b64b261..2f5b2adfc 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -67,6 +67,7 @@ module "gke" { { name = "default-node-pool" machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 local_ssd_count = 0 @@ -105,6 +106,18 @@ module "gke" { } } + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = true + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + node_pools_tags = { all = [] @@ -131,18 +144,22 @@ Then perform the following commands on the root folder: | add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | | firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | gcloud\_skip\_download | Whether to skip downloading gcloud (assumes gcloud is already available outside the module) | bool | `"true"` | no | @@ -150,6 +167,7 @@ Then perform the following commands on the root folder: | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`) | string | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | @@ -167,6 +185,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"GKE_METADATA_SERVER"` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | @@ -178,6 +197,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | @@ -195,6 +215,7 @@ Then perform the following commands on the root folder: | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity namespace | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -208,6 +229,7 @@ Then perform the following commands on the root folder: | node\_pools\_versions | List of node pools versions | | peering\_name | The name of the peering between this cluster and the Google owned VPC. | | region | Cluster region | +| release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | @@ -228,17 +250,22 @@ The node_pools variable takes the following parameters: | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| effect | Effect for the taint | | Required | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| key | The key required for the taint | | Required | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusers | | Required | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | +| node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | | tags | The list of instance tags applied to all nodes | | Required | +| value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | @@ -258,7 +285,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v2.9 +- [Terraform Provider for GCP][terraform-provider-google] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index f624fe1e7..3bfc09c8d 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -41,17 +41,37 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { + for_each = local.release_channel + + content { + channel = release_channel.value.channel + } + } subnetwork = "projects/${local.network_project_id}/regions/${var.region}/subnetworks/${var.subnetwork}" - min_master_version = local.master_version + min_master_version = var.release_channel != null ? null : local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes + enable_binary_authorization = var.enable_binary_authorization dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -115,6 +135,14 @@ resource "google_container_cluster" "primary" { node_config { service_account = lookup(var.node_pools[0], "service_account", local.service_account) + + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } @@ -149,6 +177,24 @@ resource "google_container_cluster" "primary" { } remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } + + dynamic "workload_identity_config" { + for_each = local.cluster_workload_identity_config + + content { + identity_namespace = workload_identity_config.value.identity_namespace + } + } + } /****************************************** @@ -232,6 +278,8 @@ resource "google_container_node_pool" "pools" { name = { for k, v in random_id.name : k => v.hex }[each.key] project = var.project_id location = local.location + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null cluster = google_container_cluster.primary.name @@ -327,6 +375,14 @@ resource "google_container_node_pool" "pools" { } ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) + } + } + shielded_instance_config { enable_secure_boot = lookup(each.value, "enable_secure_boot", false) enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true) diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 87bd0f47a..d645f2554 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -46,6 +46,17 @@ locals { node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 @@ -67,6 +78,9 @@ locals { provider = null }] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -114,6 +128,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] } diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index e88ca49e3..434ef2f36 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -119,6 +119,19 @@ output "service_account" { value = local.service_account } +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} + output "master_ipv4_cidr_block" { description = "The IP range in CIDR notation used for the hosted master network" value = var.master_ipv4_cidr_block diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 5df2c4afc..ca88ece9b 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -188,6 +188,24 @@ variable "enable_resource_consumption_export" { default = true } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" @@ -356,6 +374,44 @@ variable "master_ipv4_cidr_block" { default = "10.0.0.0/28" } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + +variable "release_channel" { + type = string + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + default = null +} + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster" + default = true +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index fe4eca140..2d8d7e605 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -45,6 +45,7 @@ module "gke" { { name = "default-node-pool" machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 local_ssd_count = 0 @@ -83,6 +84,18 @@ module "gke" { } } + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = true + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + node_pools_tags = { all = [] @@ -109,18 +122,22 @@ Then perform the following commands on the root folder: | add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | bool | `"false"` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | bool | `"true"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | | firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | gcloud\_skip\_download | Whether to skip downloading gcloud (assumes gcloud is already available outside the module) | bool | `"true"` | no | @@ -128,6 +145,7 @@ Then perform the following commands on the root folder: | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`) | string | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | @@ -145,6 +163,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"GKE_METADATA_SERVER"` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | @@ -156,6 +175,7 @@ Then perform the following commands on the root folder: | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | string | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | @@ -173,6 +193,7 @@ Then perform the following commands on the root folder: | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity namespace | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -186,6 +207,7 @@ Then perform the following commands on the root folder: | node\_pools\_versions | List of node pools versions | | peering\_name | The name of the peering between this cluster and the Google owned VPC. | | region | Cluster region | +| release\_channel | The release channel of this cluster | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | @@ -206,17 +228,22 @@ The node_pools variable takes the following parameters: | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| effect | Effect for the taint | | Required | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| key | The key required for the taint | | Required | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusers | | Required | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | +| node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | | tags | The list of instance tags applied to all nodes | | Required | +| value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | @@ -236,7 +263,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v2.9 +- [Terraform Provider for GCP][terraform-provider-google] v3.41 ### Configure a Service Account In order to execute this module you must have a Service Account with the diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index a0ed848c9..5ddbea5b9 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -41,17 +41,37 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { + for_each = local.release_channel + + content { + channel = release_channel.value.channel + } + } subnetwork = "projects/${local.network_project_id}/regions/${var.region}/subnetworks/${var.subnetwork}" - min_master_version = local.master_version + min_master_version = var.release_channel != null ? null : local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } default_max_pods_per_node = var.default_max_pods_per_node + enable_shielded_nodes = var.enable_shielded_nodes + enable_binary_authorization = var.enable_binary_authorization dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -115,6 +135,14 @@ resource "google_container_cluster" "primary" { node_config { service_account = lookup(var.node_pools[0], "service_account", local.service_account) + + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } @@ -149,6 +177,24 @@ resource "google_container_cluster" "primary" { } remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } + + dynamic "workload_identity_config" { + for_each = local.cluster_workload_identity_config + + content { + identity_namespace = workload_identity_config.value.identity_namespace + } + } + } /****************************************** @@ -160,6 +206,8 @@ resource "google_container_node_pool" "pools" { name = each.key project = var.project_id location = local.location + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(each.value, "node_locations", "") != "" ? split(",", each.value["node_locations"]) : null cluster = google_container_cluster.primary.name @@ -255,6 +303,14 @@ resource "google_container_node_pool" "pools" { } ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = lookup(each.value, "node_metadata", workload_metadata_config.value.node_metadata) + } + } + shielded_instance_config { enable_secure_boot = lookup(each.value, "enable_secure_boot", false) enable_integrity_monitoring = lookup(each.value, "enable_integrity_monitoring", true) diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 87bd0f47a..d645f2554 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -46,6 +46,17 @@ locals { node_pool_names = [for np in toset(var.node_pools) : np.name] node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 @@ -67,6 +78,9 @@ locals { provider = null }] + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_output_name = google_container_cluster.primary.name cluster_output_regional_zones = google_container_cluster.primary.node_locations @@ -114,6 +128,10 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + workload_identity_enabled = ! (var.identity_namespace == null || var.identity_namespace == "null") + cluster_workload_identity_config = ! local.workload_identity_enabled ? [] : var.identity_namespace == "enabled" ? [{ + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace + }] } diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index e88ca49e3..434ef2f36 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -119,6 +119,19 @@ output "service_account" { value = local.service_account } +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} + output "master_ipv4_cidr_block" { description = "The IP range in CIDR notation used for the hosted master network" value = var.master_ipv4_cidr_block diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 5df2c4afc..ca88ece9b 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -188,6 +188,24 @@ variable "enable_resource_consumption_export" { default = true } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" @@ -356,6 +374,44 @@ variable "master_ipv4_cidr_block" { default = "10.0.0.0/28" } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + +variable "release_channel" { + type = string + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + default = null +} + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster" + default = true +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool diff --git a/outputs.tf b/outputs.tf index 64263516b..9677c7c4c 100644 --- a/outputs.tf +++ b/outputs.tf @@ -118,3 +118,16 @@ output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." value = local.service_account } + +output "release_channel" { + description = "The release channel of this cluster" + value = var.release_channel +} + +output "identity_namespace" { + description = "Workload Identity namespace" + value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].identity_namespace : null + depends_on = [ + google_container_cluster.primary + ] +} diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf index 2e08ae662..cb50faf35 100644 --- a/test/fixtures/simple_regional/example.tf +++ b/test/fixtures/simple_regional/example.tf @@ -26,4 +26,5 @@ module "example" { ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_accounts[0] skip_provisioners = true + enable_binary_authorization = true } diff --git a/test/integration/simple_regional/controls/gcloud.rb b/test/integration/simple_regional/controls/gcloud.rb index bc92583e8..ba3002133 100644 --- a/test/integration/simple_regional/controls/gcloud.rb +++ b/test/integration/simple_regional/controls/gcloud.rb @@ -53,6 +53,24 @@ "networkPolicyConfig" => {}, ) end + + it "has the expected databaseEncryption config" do + expect(data['databaseEncryption']).to eq({ + "state" => 'DECRYPTED', + }) + end + + it "has the expected shieldedNodes config" do + expect(data['shieldedNodes']).to eq({ + "enabled" => true, + }) + end + + it "has the expected binaryAuthorization config" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end end describe "default node pool" do diff --git a/variables.tf b/variables.tf index e197d422e..9804cb2b7 100644 --- a/variables.tf +++ b/variables.tf @@ -188,6 +188,24 @@ variable "enable_resource_consumption_export" { default = true } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" @@ -332,6 +350,44 @@ variable "default_max_pods_per_node" { default = 110 } +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "GKE_METADATA_SERVER" + type = string +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "identity_namespace" { + description = "Workload Identity namespace. (Default value of `enabled` automatically sets project based namespace `[project_id].svc.id.goog`)" + type = string + default = "enabled" +} + +variable "release_channel" { + type = string + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." + default = null +} + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster" + default = true +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} variable "add_cluster_firewall_rules" { type = bool