diff --git a/.kitchen.yml b/.kitchen.yml index 7f136310b..a48a53aca 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -80,6 +80,13 @@ suites: systems: - name: simple_regional_private backend: local + - name: "simple_regional_with_gateway_api" + driver: + root_module_directory: test/fixtures/simple_regional_with_gateway_api + verifier: + systems: + - name: simple_regional_with_gateway_api + backend: local - name: "simple_regional_with_kubeconfig" driver: root_module_directory: test/fixtures/simple_regional_with_kubeconfig diff --git a/README.md b/README.md index d740908b6..bb1208e21 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -215,6 +216,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -302,8 +304,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP][terraform-provider-google] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/autogen/main/README.md b/autogen/main/README.md index 2d79b07ea..4f1a2933c 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -254,11 +254,11 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ {% if beta_cluster %} -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 {% else %} -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform Provider for GCP][terraform-provider-google] v4.47 {% endif %} #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 68476199f..6ecb3906f 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -53,6 +53,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 109146110..10d09a3d4 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -57,6 +57,7 @@ locals { {% endif %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] {% if autopilot_cluster != true %} autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 9c002d0fd..d22e8bd08 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -158,6 +158,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 7d57c4d74..f711f9de2 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -454,6 +454,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 70f2791b6..092d2fe75 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -38,7 +38,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.45.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index f48a35e4d..f9cc9a6f5 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -121,6 +121,21 @@ steps: - verify simple-regional-with-kubeconfig-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-kubeconfig-local'] +- id: converge simple-regional-with-gateway-api-local + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-with-gateway-api-local'] +- id: verify simple-regional-with-gateway-api-local + waitFor: + - converge simple-regional-with-gateway-api-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-with-gateway-api-local'] +- id: destroy simple-regional-with-gateway-api-local + waitFor: + - verify simple-regional-with-gateway-api-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-gateway-api-local'] - id: converge simple-regional-with-networking-local waitFor: - create all diff --git a/cluster.tf b/cluster.tf index eeccd2b2a..acf5d8095 100644 --- a/cluster.tf +++ b/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/examples/simple_regional_with_gateway_api/README.md b/examples/simple_regional_with_gateway_api/README.md new file mode 100644 index 000000000..ca82e650c --- /dev/null +++ b/examples/simple_regional_with_gateway_api/README.md @@ -0,0 +1,47 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple cluster. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | +| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | +| network | The VPC network to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region to host the cluster in | `any` | n/a | yes | +| skip\_provisioners | Flag to skip local-exec provisioners | `bool` | `false` | no | +| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | n/a | +| client\_token | n/a | +| cluster\_name | Cluster name | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | n/a | +| location | n/a | +| master\_kubernetes\_version | The master Kubernetes version | +| network | n/a | +| project\_id | n/a | +| region | n/a | +| service\_account | The default service account used for running nodes. | +| subnetwork | n/a | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_with_gateway_api/main.tf b/examples/simple_regional_with_gateway_api/main.tf new file mode 100644 index 000000000..96e9c126c --- /dev/null +++ b/examples/simple_regional_with_gateway_api/main.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-gatewayapi" +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "../../" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + enable_cost_allocation = true + enable_binary_authorization = var.enable_binary_authorization + skip_provisioners = var.skip_provisioners + gateway_api_channel = var.gateway_api_channel +} diff --git a/examples/simple_regional_with_gateway_api/outputs.tf b/examples/simple_regional_with_gateway_api/outputs.tf new file mode 100644 index 000000000..01a13147c --- /dev/null +++ b/examples/simple_regional_with_gateway_api/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + diff --git a/examples/simple_regional_with_gateway_api/test_outputs.tf b/examples/simple_regional_with_gateway_api/test_outputs.tf new file mode 120000 index 000000000..17b34213b --- /dev/null +++ b/examples/simple_regional_with_gateway_api/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/simple_regional_with_gateway_api/variables.tf b/examples/simple_regional_with_gateway_api/variables.tf new file mode 100644 index 000000000..90ba0ea0f --- /dev/null +++ b/examples/simple_regional_with_gateway_api/variables.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for services" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} + +variable "skip_provisioners" { + type = bool + description = "Flag to skip local-exec provisioners" + default = false +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} + +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} diff --git a/examples/simple_regional_with_gateway_api/versions.tf b/examples/simple_regional_with_gateway_api/versions.tf new file mode 100644 index 000000000..e8fbb1aad --- /dev/null +++ b/examples/simple_regional_with_gateway_api/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/main.tf b/main.tf index 37a6f70ea..f242aa8a4 100644 --- a/main.tf +++ b/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 5b0e80e36..d7a00acbd 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -92,6 +92,7 @@ Then perform the following commands on the root folder: | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -143,6 +144,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -185,8 +187,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index 7d5614fe7..bbf573016 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -39,6 +39,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index 18d889a0e..8e1964224 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -45,7 +45,8 @@ locals { master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index c0a333f39..a56e4b4fa 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -129,6 +129,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 3be57a3fa..896bea800 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -323,6 +323,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 274dc7faa..addb5d5b8 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -83,6 +83,7 @@ Then perform the following commands on the root folder: | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -132,6 +133,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -172,8 +174,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index a1ab9cd62..8a127f88e 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -39,6 +39,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index 652b423f7..6ec8dbcd6 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -45,7 +45,8 @@ locals { master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index 5ce92e94d..533f81884 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -129,6 +129,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index e4f4bac2a..61304cf07 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -293,6 +293,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index c2f4a90c7..840550f7d 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -201,6 +201,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | @@ -274,6 +275,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -376,8 +378,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index e9cbce191..68a5b8a49 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index cc14d3590..c7ad1f3c9 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index afc6c5561..abdf16f90 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 958f589b4..163ca7529 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -436,6 +436,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 6e71f4cae..40606fd57 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -179,6 +179,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | @@ -252,6 +253,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -354,8 +356,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 0040d5d9b..f2aeec2a6 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index cc14d3590..c7ad1f3c9 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index afc6c5561..abdf16f90 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 958f589b4..163ca7529 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -436,6 +436,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index e4bd8d3cc..2e5923191 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -192,6 +192,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | @@ -263,6 +264,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -363,8 +365,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 09938b2cb..fd5e4675b 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 3dd8c73e4..02195dd00 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index 31556abec..ed73acae2 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index cf768470b..9e8d5189b 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -406,6 +406,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 091e96785..1268aafbf 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -170,6 +170,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | @@ -241,6 +242,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -341,8 +343,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 8ced17225..7eb7269fb 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 3dd8c73e4..02195dd00 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 31556abec..ed73acae2 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index cf768470b..9e8d5189b 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -406,6 +406,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index bdebd7adc..03b265ace 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -186,6 +186,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -248,6 +249,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -337,8 +339,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP][terraform-provider-google] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 36cba1037..38a176fc0 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index f2f012078..3776e924f 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index c3436263f..28350e722 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 038ca7d96..e5b2a618f 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -417,6 +417,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index b91390550..8fba8fefb 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.45.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 5230da33d..dee1ca012 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -164,6 +164,7 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -226,6 +227,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -315,8 +317,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP][terraform-provider-google] v4.47 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 609962330..28378512a 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index f2f012078..3776e924f 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index c3436263f..28350e722 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 038ca7d96..e5b2a618f 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -417,6 +417,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index 0294245b7..5d0dbd548 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.45.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/outputs.tf b/outputs.tf index 7e3fb254c..ca55e48b8 100644 --- a/outputs.tf +++ b/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/test/fixtures/simple_regional_with_gateway_api/example.tf b/test/fixtures/simple_regional_with_gateway_api/example.tf new file mode 100644 index 000000000..189a65ba1 --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/example.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_index = 1 +} + +module "example" { + source = "../../../examples/simple_regional_with_gateway_api" + + project_id = var.project_ids[local.cluster_index] + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_accounts[local.cluster_index] + skip_provisioners = true + enable_binary_authorization = true + gateway_api_channel = "CHANNEL_STANDARD" +} diff --git a/test/fixtures/simple_regional_with_gateway_api/network.tf b/test/fixtures/simple_regional_with_gateway_api/network.tf new file mode 100644 index 000000000..a7ca31698 --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_ids[local.cluster_index] +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/simple_regional_with_gateway_api/outputs.tf b/test/fixtures/simple_regional_with_gateway_api/outputs.tf new file mode 120000 index 000000000..726bdc722 --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/simple_regional_with_gateway_api/variables.tf b/test/fixtures/simple_regional_with_gateway_api/variables.tf new file mode 120000 index 000000000..c113c00a3 --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/simple_regional_with_gateway_api/controls/gcloud.rb b/test/integration/simple_regional_with_gateway_api/controls/gcloud.rb new file mode 100644 index 000000000..14722175b --- /dev/null +++ b/test/integration/simple_regional_with_gateway_api/controls/gcloud.rb @@ -0,0 +1,199 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq nil + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to include( + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + ) + end + + it "has gateway api enabled" do + expect(data['networkConfig']).to include( + "gatewayApiConfig" => { + "channel" => "CHANNEL_STANDARD", + }, + ) + end + + it "has the expected databaseEncryption config" do + expect(data['databaseEncryption']).to eq({ + "state" => 'DECRYPTED', + }) + end + + it "has the expected shieldedNodes config" do + expect(data['shieldedNodes']).to eq({ + "enabled" => true, + }) + end + + it "has the expected binaryAuthorization config" do + expect(data['binaryAuthorization']).to eq({ + "evaluationMode" => "PROJECT_SINGLETON_POLICY_ENFORCE", + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "e2-medium", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + + it "has autoupgrade enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoUpgrade" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/simple_regional_with_gateway_api/inspec.yml b/test/integration/simple_regional_with_gateway_api/inspec.yml new file mode 100644 index 000000000..e91bbc6ca --- /dev/null +++ b/test/integration/simple_regional_with_gateway_api/inspec.yml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: simple_regional_with_gateway_api +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/variables.tf b/variables.tf index 13839b02f..bd0b0d927 100644 --- a/variables.tf +++ b/variables.tf @@ -387,6 +387,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" diff --git a/versions.tf b/versions.tf index d7edee28e..6d0841514 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.45.0, < 5.0" + version = ">= 4.47.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes"