From 634063ff3a9cf50e242451a01d6a559d4617aa29 Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Wed, 22 May 2024 20:51:49 +0000 Subject: [PATCH 1/2] feat: add queued_provisioning --- README.md | 1 + autogen/main/README.md | 1 + autogen/main/cluster.tf.tmpl | 14 ++++++++++ autogen/main/versions.tf.tmpl | 2 +- cluster.tf | 26 ++++++++++++++++++ examples/node_pool/main.tf | 6 +++++ .../README.md | 1 + .../cluster.tf | 27 +++++++++++++++++++ modules/beta-private-cluster/README.md | 1 + modules/beta-private-cluster/cluster.tf | 26 ++++++++++++++++++ .../README.md | 1 + .../cluster.tf | 27 +++++++++++++++++++ modules/beta-public-cluster/README.md | 1 + modules/beta-public-cluster/cluster.tf | 26 ++++++++++++++++++ .../private-cluster-update-variant/README.md | 1 + .../private-cluster-update-variant/cluster.tf | 27 +++++++++++++++++++ .../versions.tf | 2 +- modules/private-cluster/README.md | 1 + modules/private-cluster/cluster.tf | 26 ++++++++++++++++++ modules/private-cluster/versions.tf | 2 +- test/integration/node_pool/controls/gcloud.rb | 23 +++++++++++++++- versions.tf | 2 +- 22 files changed, 239 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c354cd843f..231c3e4e8f 100644 --- a/README.md +++ b/README.md @@ -329,6 +329,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/autogen/main/README.md b/autogen/main/README.md index 5801d04b4c..5d1afefafc 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -254,6 +254,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index d0c7b37162..c43d736adf 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -658,6 +658,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -811,6 +812,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -827,6 +835,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 31c1e0d04a..41ab9a47df 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -46,7 +46,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.21.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/cluster.tf b/cluster.tf index 233c4c4751..05a57c0372 100644 --- a/cluster.tf +++ b/cluster.tf @@ -536,6 +536,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -552,6 +559,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -774,6 +787,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -790,6 +810,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index e00e9c5136..7626216fb7 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -80,6 +80,12 @@ module "gke" { cpu_cfs_quota = true local_ssd_ephemeral_count = 2 }, + { + name = "pool-04" + min_count = 0 + service_account = var.compute_engine_service_account + queued_provisioning = true + }, ] node_pools_metadata = { diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 883af3419e..d82abcecc7 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -395,6 +395,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index f39cc552e7..ff813fabf7 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -567,6 +567,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -697,6 +698,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -713,6 +721,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -961,6 +975,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -977,6 +998,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 0f06cabef4..2d9d7ff617 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -373,6 +373,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index cce6101889..bfd0bbc3e4 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -621,6 +621,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -637,6 +644,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -884,6 +897,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -900,6 +920,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 12dacc2128..49e6e4b284 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -383,6 +383,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index edfd3e161f..1045fca43c 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -548,6 +548,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -678,6 +679,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -694,6 +702,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -942,6 +956,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -958,6 +979,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 5a5ce13dea..557b7dfcee 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -361,6 +361,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index dfecb9e86d..aae96d3dda 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -602,6 +602,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -618,6 +625,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -865,6 +878,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -881,6 +901,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index b20d8da1af..3b3e31d4c1 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -364,6 +364,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 29d37b6635..762c09897f 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -501,6 +501,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -631,6 +632,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -647,6 +655,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -870,6 +884,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -886,6 +907,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index 0a54060183..0eba349492 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.21.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 9d77ba0c24..7b03a27e40 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -342,6 +342,7 @@ The node_pools variable takes the following parameters: | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | | secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 14da218587..3b82b1b4a5 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -555,6 +555,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -571,6 +578,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -793,6 +806,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -809,6 +829,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index 4e00391ac0..473918056d 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.21.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 38ac3ef554..d6ff8202f1 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -62,7 +62,7 @@ describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" || p['name'] =~ %r{^nap-.*} } } - it "has 3" do + it "has 4" do expect(node_pools.count).to eq 3 end @@ -493,6 +493,27 @@ ) end end + + describe "pool-04" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "pool-04", + ) + ) + end + + it "has queued_provisioning enabled" do + expect(data['nodePools']).not_to include( + including( + "name" => "pool-04", + "queued_provisioning" => including( + "enabled" => true, + ), + ) + ) + end + end end end diff --git a/versions.tf b/versions.tf index 0495aabdf5..2298d08be8 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.21.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" From 99f89d6add7357f461e0208f0577f47f88061ce3 Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Fri, 24 May 2024 10:36:08 -0700 Subject: [PATCH 2/2] Update gcloud.rb --- test/integration/node_pool/controls/gcloud.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index d6ff8202f1..e3da9d06b2 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -63,7 +63,7 @@ let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" || p['name'] =~ %r{^nap-.*} } } it "has 4" do - expect(node_pools.count).to eq 3 + expect(node_pools.count).to eq 4 end describe "pool-01" do