From 129116cc252d7307cdb0e4f020dcc7b531c6eea1 Mon Sep 17 00:00:00 2001 From: James Timms Date: Wed, 30 Jun 2021 15:42:17 +0100 Subject: [PATCH 1/7] feature: add gpu node autoscaling support (#807) * add gpu node autoscaling support for top level module * add gpu node autoscaling support for beta-private-cluster module --- main.tf | 4 ++-- modules/beta-private-cluster/main.tf | 4 ++-- modules/beta-private-cluster/variables.tf | 6 ++++++ variables.tf | 6 ++++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index c566719500..c232dd2e23 100644 --- a/main.tf +++ b/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 0219f1da8f..a24a0b0107 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 26ac028fab..9ca2b4c441 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -241,6 +241,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -249,6 +254,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/variables.tf b/variables.tf index 95cc1c63d7..e7d8f8032c 100644 --- a/variables.tf +++ b/variables.tf @@ -207,6 +207,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -214,6 +219,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } From 37e004738155b8bcad74f4f7be1ad94dba5108cf Mon Sep 17 00:00:00 2001 From: James Timms Date: Wed, 30 Jun 2021 16:47:33 +0100 Subject: [PATCH 2/7] feature: add gpu node autoscaling support for all modules (#807) * add gpu node autoscaling support for all modules --- README.md | 2 +- autogen/main/main.tf.tmpl | 4 ++-- autogen/main/variables.tf.tmpl | 6 ++++++ modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster-update-variant/main.tf | 4 ++-- modules/beta-private-cluster-update-variant/variables.tf | 6 ++++++ modules/beta-private-cluster/README.md | 2 +- modules/beta-private-cluster/variables.tf | 2 +- modules/beta-public-cluster-update-variant/README.md | 2 +- modules/beta-public-cluster-update-variant/main.tf | 4 ++-- modules/beta-public-cluster-update-variant/variables.tf | 6 ++++++ modules/beta-public-cluster/README.md | 2 +- modules/beta-public-cluster/main.tf | 4 ++-- modules/beta-public-cluster/variables.tf | 6 ++++++ modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster-update-variant/main.tf | 4 ++-- modules/private-cluster-update-variant/variables.tf | 6 ++++++ modules/private-cluster/README.md | 2 +- modules/private-cluster/main.tf | 4 ++-- modules/private-cluster/variables.tf | 6 ++++++ 20 files changed, 56 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 958c0c744c..6a0e52993f 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
  })
| 
{
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index e219e9139b..787c860a92 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -55,7 +55,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -63,7 +63,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index d146473392..e46a69861c 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -251,6 +251,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -261,6 +266,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 276b803e84..b0dac09d16 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -162,7 +162,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 0219f1da8f..a24a0b0107 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 26ac028fab..e971004de0 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -241,6 +241,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -249,6 +254,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 64e864f89d..467e2ee05d 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -140,7 +140,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 9ca2b4c441..e971004de0 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -241,7 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ + gpu_resources = list(object({ resource_type = string minimum = number maximum = number diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 3eb33505ab..1cdcee231a 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -156,7 +156,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 9d72a02845..fce781a0e0 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 778660396b..3efac54988 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -241,6 +241,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -249,6 +254,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7f49d35170..24fc286dba 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -134,7 +134,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9d72a02845..fce781a0e0 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 778660396b..3efac54988 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -241,6 +241,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -249,6 +254,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 79a1560aff..422f86cf56 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -155,7 +155,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
  })
| 
{
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index a31a7ee708..54daa95ba2 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 38ae5b7fd0..d02341b844 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -207,6 +207,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -214,6 +219,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 3f5b8b8c14..15402b153a 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -133,7 +133,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
  })
| 
{
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index a31a7ee708..54daa95ba2 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -51,7 +51,7 @@ locals { release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - autoscaling_resource_limits = var.cluster_autoscaling.enabled ? [{ + autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" minimum = var.cluster_autoscaling.min_cpu_cores maximum = var.cluster_autoscaling.max_cpu_cores @@ -59,7 +59,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + }], var.cluster_autoscaling.gpu_resources) : [] custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 38ae5b7fd0..d02341b844 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -207,6 +207,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -214,6 +219,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } From bb7ce1cec907ca036f6833cdaf32f2bf66318316 Mon Sep 17 00:00:00 2001 From: James Timms Date: Wed, 30 Jun 2021 17:37:13 +0100 Subject: [PATCH 3/7] feature: add gpu node autoscaling support for all modules (#807) * updater example/node_pool cluster_autoscaling var to work with gpu_resources --- examples/node_pool/variables.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index e46f8f927a..d2980a3929 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -60,6 +60,11 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number + gpu_resources = list(object({ + resource_type = string + minimum = number + maximum = number + })) }) default = { enabled = false @@ -68,6 +73,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } From 4b0c93144d5b5e4dd7599e1fbf213b5f33295f32 Mon Sep 17 00:00:00 2001 From: James Timms Date: Wed, 30 Jun 2021 17:56:36 +0100 Subject: [PATCH 4/7] feature: add gpu node autoscaling support for all modules (#807) * fix example/node_pool formatting error --- examples/node_pool/README.md | 2 +- examples/node_pool/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index ec20ffa877..3d2b13102c 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,7 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index d2980a3929..ac76aa1a05 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -60,7 +60,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ + gpu_resources = list(object({ resource_type = string minimum = number maximum = number From 5f5dbf6cdbbba7e83192e7e32f77ea0536579364 Mon Sep 17 00:00:00 2001 From: James Timms Date: Sun, 4 Jul 2021 11:05:16 +0100 Subject: [PATCH 5/7] feature: add gpu node autoscaling support for all modules (#807) * Format gpu_resource to meet linter requirements * Update examples/node_pool/ --- README.md | 2 +- autogen/main/variables.tf.tmpl | 8 ++------ examples/node_pool/README.md | 2 +- modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster-update-variant/variables.tf | 6 +----- modules/beta-private-cluster/README.md | 2 +- modules/beta-private-cluster/variables.tf | 6 +----- modules/beta-public-cluster-update-variant/README.md | 2 +- modules/beta-public-cluster-update-variant/variables.tf | 6 +----- modules/beta-public-cluster/README.md | 2 +- modules/beta-public-cluster/variables.tf | 6 +----- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster-update-variant/variables.tf | 6 +----- modules/private-cluster/README.md | 2 +- modules/private-cluster/variables.tf | 6 +----- variables.tf | 6 +----- 16 files changed, 17 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 6a0e52993f..153bedc4c7 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index e46a69861c..52013bd388 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -251,11 +251,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false @@ -266,7 +262,7 @@ variable "cluster_autoscaling" { min_cpu_cores = 0 max_memory_gb = 0 min_memory_gb = 0 - gpu_resources = [] + gpu_resources = [] } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 3d2b13102c..48d27a1e9c 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,7 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index b0dac09d16..7e3a4562df 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -162,7 +162,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index e971004de0..3ff81e17b1 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -241,11 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 467e2ee05d..5a8032b5f8 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -140,7 +140,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index e971004de0..3ff81e17b1 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -241,11 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 1cdcee231a..c289633621 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -156,7 +156,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 3efac54988..e805b0bd22 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -241,11 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 24fc286dba..4e5a78ee37 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -134,7 +134,7 @@ Then perform the following commands on the root folder: | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 3efac54988..e805b0bd22 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -241,11 +241,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 422f86cf56..66d82b078c 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -155,7 +155,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index d02341b844..a299d887f3 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -207,11 +207,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 15402b153a..24b807c6b3 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -133,7 +133,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `any` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index d02341b844..a299d887f3 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -207,11 +207,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false diff --git a/variables.tf b/variables.tf index e7d8f8032c..c6a654cb02 100644 --- a/variables.tf +++ b/variables.tf @@ -207,11 +207,7 @@ variable "cluster_autoscaling" { max_cpu_cores = number min_memory_gb = number max_memory_gb = number - gpu_resources = list(object({ - resource_type = string - minimum = number - maximum = number - })) + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) }) default = { enabled = false From bfa8bf417105b0bb444f62db553c9870e218f77a Mon Sep 17 00:00:00 2001 From: James Timms Date: Sun, 4 Jul 2021 11:40:54 +0100 Subject: [PATCH 6/7] feature: add gpu node autoscaling support for all modules (#807) * Add v16.0 upgrade guide * Update node_pool test to specify `gpu_resources` --- docs/upgrading_to_v16.0.md | 28 ++++++++++++++++++++++++++++ test/fixtures/node_pool/example.tf | 1 + 2 files changed, 29 insertions(+) create mode 100644 docs/upgrading_to_v16.0.md diff --git a/docs/upgrading_to_v16.0.md b/docs/upgrading_to_v16.0.md new file mode 100644 index 0000000000..e18ffafd10 --- /dev/null +++ b/docs/upgrading_to_v16.0.md @@ -0,0 +1,28 @@ +# Upgrading to v16.0 + +The v16.0 release of *kubernetes-engine* is a backwards incompatible release. + +### cluster_autoscaling modified +The `cluster_autoscaling` variable has been modified to require a `gpu_resources` value. + +```diff + module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" +- version = "~> 15.0" ++ version = "~> 16.0" + + cluster_autoscaling = { + enabled = true + autoscaling_profile = "BALANCED" + min_cpu_cores = 1 + max_cpu_cores = 100 + min_memory_gb = 1 + max_memory_gb = 1000 ++ gpu_resources = [{ ++ resource_type = "nvidia-tesla-t4" ++ minimum = 1 ++ maximum = 3 ++ }] + } +} +``` diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index f2298c3865..c5741bdbed 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -34,6 +34,7 @@ module "example" { min_cpu_cores = 5 max_memory_gb = 30 min_memory_gb = 10 + gpu_resources = [] } } From 934a06d94993e34a6ffcd951a0ec56c17affe7a3 Mon Sep 17 00:00:00 2001 From: James Timms Date: Thu, 8 Jul 2021 17:22:38 +0100 Subject: [PATCH 7/7] feature: add gpu node autoscaling support for all modules (#807) * updates upgrade guide Co-authored-by: Bharath KKB --- docs/upgrading_to_v16.0.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/upgrading_to_v16.0.md b/docs/upgrading_to_v16.0.md index e18ffafd10..8c577b5859 100644 --- a/docs/upgrading_to_v16.0.md +++ b/docs/upgrading_to_v16.0.md @@ -3,7 +3,7 @@ The v16.0 release of *kubernetes-engine* is a backwards incompatible release. ### cluster_autoscaling modified -The `cluster_autoscaling` variable has been modified to require a `gpu_resources` value. +The `cluster_autoscaling` variable has been modified to require a `gpu_resources` value. If you have enabled `cluster_autoscaling` and do not require `gpu_resources`, you can set it to an empty list as shown below. ```diff module "gke" { @@ -18,11 +18,7 @@ The `cluster_autoscaling` variable has been modified to require a `gpu_resources max_cpu_cores = 100 min_memory_gb = 1 max_memory_gb = 1000 -+ gpu_resources = [{ -+ resource_type = "nvidia-tesla-t4" -+ minimum = 1 -+ maximum = 3 -+ }] ++ gpu_resources = [] } } ```