From e53a949347417c1025eb433bc61c2a3e62e3df1c Mon Sep 17 00:00:00 2001 From: James Timms Date: Fri, 9 Jul 2021 04:02:54 +0100 Subject: [PATCH] feat!: add gpu node autoscaling support (#807) (#944) * feature: add gpu node autoscaling support (#807) * add gpu node autoscaling support for top level module * add gpu node autoscaling support for beta-private-cluster module * feature: add gpu node autoscaling support for all modules (#807) * add gpu node autoscaling support for all modules * feature: add gpu node autoscaling support for all modules (#807) * updater example/node_pool cluster_autoscaling var to work with gpu_resources * feature: add gpu node autoscaling support for all modules (#807) * fix example/node_pool formatting error * feature: add gpu node autoscaling support for all modules (#807) * Format gpu_resource to meet linter requirements * Update examples/node_pool/ * feature: add gpu node autoscaling support for all modules (#807) * Add v16.0 upgrade guide * Update node_pool test to specify `gpu_resources` * feature: add gpu node autoscaling support for all modules (#807) * updates upgrade guide Co-authored-by: Bharath KKB Co-authored-by: Bharath KKB --- README.md | 2 +- autogen/main/main.tf.tmpl | 4 ++-- autogen/main/variables.tf.tmpl | 2 ++ docs/upgrading_to_v16.0.md | 24 +++++++++++++++++++ examples/node_pool/README.md | 2 +- examples/node_pool/variables.tf | 6 +++++ main.tf | 4 ++-- .../README.md | 2 +- .../main.tf | 4 ++-- .../variables.tf | 2 ++ modules/beta-private-cluster/README.md | 2 +- modules/beta-private-cluster/main.tf | 4 ++-- modules/beta-private-cluster/variables.tf | 2 ++ .../README.md | 2 +- .../main.tf | 4 ++-- .../variables.tf | 2 ++ modules/beta-public-cluster/README.md | 2 +- modules/beta-public-cluster/main.tf | 4 ++-- modules/beta-public-cluster/variables.tf | 2 ++ .../private-cluster-update-variant/README.md | 2 +- .../private-cluster-update-variant/main.tf | 4 ++-- .../variables.tf | 2 ++ modules/private-cluster/README.md | 2 +- modules/private-cluster/main.tf | 4 ++-- modules/private-cluster/variables.tf | 2 ++ test/fixtures/node_pool/example.tf | 1 + variables.tf | 2 ++ 27 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 docs/upgrading_to_v16.0.md diff --git a/README.md b/README.md index 958c0c744..153bedc4c 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
  })
| 
{
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index e219e9139..787c860a9 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -55,7 +55,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -63,7 +63,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index d14647339..52013bd38 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -251,6 +251,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -261,6 +262,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/docs/upgrading_to_v16.0.md b/docs/upgrading_to_v16.0.md new file mode 100644 index 000000000..8c577b585 --- /dev/null +++ b/docs/upgrading_to_v16.0.md @@ -0,0 +1,24 @@ +# Upgrading to v16.0 + +The v16.0 release of *kubernetes-engine* is a backwards incompatible release. + +### cluster_autoscaling modified +The `cluster_autoscaling` variable has been modified to require a `gpu_resources` value. If you have enabled `cluster_autoscaling` and do not require `gpu_resources`, you can set it to an empty list as shown below. + +```diff + module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" +- version = "~> 15.0" ++ version = "~> 16.0" + + cluster_autoscaling = { + enabled = true + autoscaling_profile = "BALANCED" + min_cpu_cores = 1 + max_cpu_cores = 100 + min_memory_gb = 1 + max_memory_gb = 1000 ++ gpu_resources = [] + } +} +``` diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index ec20ffa87..48d27a1e9 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,7 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index e46f8f927..ac76aa1a0 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -60,6 +60,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -68,6 +73,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/main.tf b/main.tf index c56671950..c232dd2e2 100644 --- a/main.tf +++ b/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 276b803e8..7e3a4562d 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -162,7 +162,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 0219f1da8..a24a0b010 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 26ac028fa..3ff81e17b 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -241,6 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -249,6 +250,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 64e864f89..5a8032b5f 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -140,7 +140,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 0219f1da8..a24a0b010 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 26ac028fa..3ff81e17b 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -241,6 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -249,6 +250,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 3eb33505a..c28963362 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -156,7 +156,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 9d72a0284..fce781a0e 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 778660396..e805b0bd2 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -241,6 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -249,6 +250,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7f49d3517..4e5a78ee3 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -134,7 +134,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9d72a0284..fce781a0e 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 778660396..e805b0bd2 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -241,6 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -249,6 +250,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 79a1560af..66d82b078 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -155,7 +155,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
  })
| 
{
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index a31a7ee70..54daa95ba 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 38ae5b7fd..a299d887f 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -207,6 +207,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -214,6 +215,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 3f5b8b8c1..24b807c6b 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -133,7 +133,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
  })
| 
{
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index a31a7ee70..54daa95ba 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 38ae5b7fd..a299d887f 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -207,6 +207,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -214,6 +215,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index f2298c386..c5741bdbe 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -34,6 +34,7 @@ module "example" { min_cpu_cores = 5 max_memory_gb = 30 min_memory_gb = 10 + gpu_resources = [] } } diff --git a/variables.tf b/variables.tf index 95cc1c63d..c6a654cb0 100644 --- a/variables.tf +++ b/variables.tf @@ -207,6 +207,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -214,6 +215,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" }