From 3db27112960c4a6669d671e68c6a5fec71e0de06 Mon Sep 17 00:00:00 2001 From: anoxape <2760053+anoxape@users.noreply.github.com> Date: Mon, 20 Jun 2022 00:23:19 -0700 Subject: [PATCH] feat: support gVNIC --- README.md | 2 ++ autogen/main/README.md | 2 ++ autogen/main/cluster.tf.tmpl | 14 ++++++++++++++ autogen/main/versions.tf.tmpl | 2 +- cluster.tf | 13 +++++++++++++ .../beta-private-cluster-update-variant/README.md | 2 ++ .../beta-private-cluster-update-variant/cluster.tf | 14 ++++++++++++++ modules/beta-private-cluster/README.md | 2 ++ modules/beta-private-cluster/cluster.tf | 13 +++++++++++++ .../beta-public-cluster-update-variant/README.md | 2 ++ .../beta-public-cluster-update-variant/cluster.tf | 14 ++++++++++++++ modules/beta-public-cluster/README.md | 2 ++ modules/beta-public-cluster/cluster.tf | 13 +++++++++++++ modules/private-cluster-update-variant/README.md | 2 ++ modules/private-cluster-update-variant/cluster.tf | 14 ++++++++++++++ modules/private-cluster-update-variant/versions.tf | 2 +- modules/private-cluster/README.md | 2 ++ modules/private-cluster/cluster.tf | 13 +++++++++++++ modules/private-cluster/versions.tf | 2 +- versions.tf | 2 +- 20 files changed, 128 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b1f8247a36..34274f59da 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -238,6 +239,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/autogen/main/README.md b/autogen/main/README.md index a41e0b6006..1a83b319d7 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -109,6 +109,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -196,6 +197,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 7892e9b0f7..ea65a70d01 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -311,6 +311,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -449,6 +456,7 @@ locals { {% endif %} "service_account", "enable_gcfs", + "enable_gvnic", "enable_secure_boot", ] } @@ -600,6 +608,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 47767df66b..71128748f7 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -38,7 +38,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.10.0, < 5.0" + version = ">= 4.14.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/cluster.tf b/cluster.tf index 3cd0f6f835..e180afeb3e 100644 --- a/cluster.tf +++ b/cluster.tf @@ -181,6 +181,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -307,6 +314,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index b75726f7a7..fbaaeb9e0f 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -92,6 +92,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -313,6 +314,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 65d0f2530f..ac589a7faf 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -274,6 +274,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -394,6 +401,7 @@ locals { "spot", "service_account", "enable_gcfs", + "enable_gvnic", "enable_secure_boot", ] } @@ -531,6 +539,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index a55d71e273..af7801ef69 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -70,6 +70,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -291,6 +292,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index b3e1c8db5c..ad630ef7c5 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -274,6 +274,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -443,6 +450,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 2810e838a3..39c87f0ec6 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -86,6 +86,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -300,6 +301,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 0f7d3b7cd6..8fb29d2f5b 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -274,6 +274,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -375,6 +382,7 @@ locals { "spot", "service_account", "enable_gcfs", + "enable_gvnic", "enable_secure_boot", ] } @@ -512,6 +520,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 6e7ad6c702..1cec3ada38 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -64,6 +64,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -278,6 +279,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 32d22347de..012ce31570 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -274,6 +274,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -424,6 +431,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index a3b0a135e8..d47e854526 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -87,6 +87,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -272,6 +273,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 1a4cdf5ebc..b4765fb489 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -181,6 +181,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -280,6 +287,7 @@ locals { "preemptible", "service_account", "enable_gcfs", + "enable_gvnic", "enable_secure_boot", ] } @@ -407,6 +415,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index 6af987847e..044695f6a1 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.10.0, < 5.0" + version = ">= 4.14.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 3ded26b96e..370faefe66 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -65,6 +65,7 @@ module "gke" { disk_type = "pd-standard" image_type = "COS_CONTAINERD" enable_gcfs = false + enable_gvnic = false auto_repair = true auto_upgrade = true service_account = "project-service-account@.iam.gserviceaccount.com" @@ -250,6 +251,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | | enable_gcfs | Google Container File System (gcfs) has to be enabled for image streaming to be active. Needs image_type to be set to COS_CONTAINERD. | false | Optional | +| enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 7b74bf1493..128ef36346 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -181,6 +181,13 @@ resource "google_container_cluster" "primary" { } } + dynamic "gvnic" { + for_each = lookup(var.node_pools[0], "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -320,6 +327,12 @@ resource "google_container_node_pool" "pools" { enabled = gcfs_config.value } } + dynamic "gvnic" { + for_each = lookup(each.value, "enable_gvnic", false) ? [true] : [] + content { + enabled = gvnic.value + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index f901705353..9567d3e227 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.10.0, < 5.0" + version = ">= 4.14.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/versions.tf b/versions.tf index 970e0413f3..e6bcafb3f4 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.10.0, < 5.0" + version = ">= 4.14.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes"