From 0eb1e16606cedb95fc7084b8c3f9c1acc9d17494 Mon Sep 17 00:00:00 2001 From: Pavel Zhukov Date: Wed, 13 Nov 2019 18:40:56 +0200 Subject: [PATCH] Make node_pools_* params optional and allow pool-specific overrides. [test/deploy_service] Add retry logic to wait until the nginx service becomes reachable [build] Add 120 seconds delay after the 'prepare' step --- Makefile | 2 +- autogen/cluster.tf.tmpl | 61 +++++++-------- autogen/main.tf.tmpl | 3 +- autogen/variables.tf.tmpl | 25 ++++--- autogen/variables_defaults.tf | 74 +++++++++++++++++++ build/int.cloudbuild.yaml | 4 +- build/lint.cloudbuild.yaml | 2 +- cluster.tf | 33 +++++---- examples/node_pool/main.tf | 16 ---- .../cluster.tf | 72 ++++++++++-------- .../main.tf | 1 + .../variables.tf | 7 ++ .../variables_defaults.tf | 72 ++++++++++++++++++ modules/beta-private-cluster/cluster.tf | 46 +++++++----- modules/beta-private-cluster/main.tf | 1 + modules/beta-private-cluster/variables.tf | 7 ++ .../variables_defaults.tf | 72 ++++++++++++++++++ modules/beta-public-cluster/cluster.tf | 46 +++++++----- modules/beta-public-cluster/main.tf | 1 + modules/beta-public-cluster/variables.tf | 7 ++ .../beta-public-cluster/variables_defaults.tf | 72 ++++++++++++++++++ .../private-cluster-update-variant/cluster.tf | 59 ++++++++------- .../variables.tf | 4 + .../variables_defaults.tf | 62 ++++++++++++++++ modules/private-cluster/cluster.tf | 33 +++++---- modules/private-cluster/variables.tf | 4 + modules/private-cluster/variables_defaults.tf | 62 ++++++++++++++++ .../deploy_service/controls/kubectl.rb | 6 ++ variables.tf | 4 + variables_defaults.tf | 62 ++++++++++++++++ 30 files changed, 739 insertions(+), 181 deletions(-) create mode 100644 autogen/variables_defaults.tf create mode 100644 modules/beta-private-cluster-update-variant/variables_defaults.tf create mode 100644 modules/beta-private-cluster/variables_defaults.tf create mode 100644 modules/beta-public-cluster/variables_defaults.tf create mode 100644 modules/private-cluster-update-variant/variables_defaults.tf create mode 100644 modules/private-cluster/variables_defaults.tf create mode 100644 variables_defaults.tf diff --git a/Makefile b/Makefile index 736cad34ce..09dc66079c 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.6 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index 27448db32d..77b3b5b84e 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -252,10 +252,10 @@ resource "random_id" "name" { labels = join(",", sort( concat( - keys(var.node_pools_labels["all"]), - values(var.node_pools_labels["all"]), - keys(var.node_pools_labels[var.node_pools[count.index]["name"]]), - values(var.node_pools_labels[var.node_pools[count.index]["name"]]) + keys(local.node_pools_labels["all"]), + values(local.node_pools_labels["all"]), + keys(local.node_pools_labels[var.node_pools[count.index]["name"]]), + values(local.node_pools_labels[var.node_pools[count.index]["name"]]) ) ) ) @@ -264,10 +264,10 @@ resource "random_id" "name" { metadata = join(",", sort( concat( - keys(var.node_pools_metadata["all"]), - values(var.node_pools_metadata["all"]), - keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]), - values(var.node_pools_metadata[var.node_pools[count.index]["name"]]) + keys(local.node_pools_metadata["all"]), + values(local.node_pools_metadata["all"]), + keys(local.node_pools_metadata[var.node_pools[count.index]["name"]]), + values(local.node_pools_metadata[var.node_pools[count.index]["name"]]) ) ) ) @@ -276,8 +276,8 @@ resource "random_id" "name" { oauth_scopes = join(",", sort( concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] ) ) ) @@ -286,8 +286,8 @@ resource "random_id" "name" { tags = join(",", sort( concat( - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]] + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]] ) ) ) @@ -314,7 +314,9 @@ resource "google_container_node_pool" "pools" { // use node_locations if provided, defaults to cluster level node_locations if not specified node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null {% endif %} - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -350,16 +352,16 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, @@ -367,8 +369,8 @@ resource "google_container_node_pool" "pools" { {% if beta_cluster %} dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -378,10 +380,10 @@ resource "google_container_node_pool" "pools" { } {% endif %} tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -396,8 +398,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -431,6 +433,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + {% if update_variant %} create_before_destroy = true {% endif %} diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 841444ea44..15d6d3983d 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -147,7 +147,8 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ + + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] # /BETA features diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index ad5cc44e34..e58f2ef487 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -79,7 +79,7 @@ variable "node_version" { } variable "master_authorized_networks_config" { - type = list(object({cidr_blocks = list(object({cidr_block = string, display_name = string}))})) + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -181,9 +183,10 @@ variable "node_pools_metadata" { {% if beta_cluster %} variable "node_pools_taints" { - type = map(list(object({key=string,value=string,effect=string}))) + type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -195,6 +198,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -205,6 +209,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -356,10 +361,11 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({state = string, key_name = string})) - default = [{ - state = "DECRYPTED" - key_name = "" + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" }] } @@ -375,7 +381,8 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." - default = [{ + + default = [{ "enabled" = false }] } @@ -429,8 +436,8 @@ variable "release_channel" { } variable "enable_shielded_nodes" { - type = bool + type = bool description = "Enable Shielded Nodes features on all nodes in this cluster" - default = false + default = false } {% endif %} diff --git a/autogen/variables_defaults.tf b/autogen/variables_defaults.tf new file mode 100644 index 0000000000..ccc9b0eed3 --- /dev/null +++ b/autogen/variables_defaults.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{{ autogeneration_note }} + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + +{% if beta_cluster %} + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + +{% endif %} + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index e78d0eb2ba..af0560641f 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -19,7 +19,7 @@ steps: args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && download_acm'] - id: prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment'] + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment && sleep 120'] env: - 'TF_VAR_org_id=$_ORG_ID' - 'TF_VAR_folder_id=$_FOLDER_ID' @@ -309,6 +309,6 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0' options: machineType: 'N1_HIGHCPU_8' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 02b9e5327b..82c43b9fa6 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -24,4 +24,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0' diff --git a/cluster.tf b/cluster.tf index 07801f226a..73d17fc26a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -131,7 +131,9 @@ resource "google_container_node_pool" "pools" { name = var.node_pools[count.index]["name"] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -164,25 +166,25 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -197,8 +199,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -214,6 +216,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 1ae41a53bb..b2f1d010ed 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -72,20 +72,10 @@ module "gke" { }, ] - node_pools_oauth_scopes = { - all = [] - pool-01 = [] - pool-02 = [] - pool-03 = [] - } - node_pools_metadata = { - all = {} pool-01 = { shutdown-script = file("${path.module}/data/shutdown-script.sh") } - pool-02 = {} - pool-03 = {} } node_pools_labels = { @@ -95,8 +85,6 @@ module "gke" { pool-01 = { pool-01-example = true } - pool-02 = {} - pool-03 = {} } node_pools_taints = { @@ -114,8 +102,6 @@ module "gke" { effect = "PREFER_NO_SCHEDULE" }, ] - pool-02 = [] - pool-03 = [] } node_pools_tags = { @@ -125,8 +111,6 @@ module "gke" { pool-01 = [ "pool-01-example", ] - pool-02 = [] - pool-03 = [] } } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 8cc2311055..ab65ff763a 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -235,10 +235,10 @@ resource "random_id" "name" { labels = join(",", sort( concat( - keys(var.node_pools_labels["all"]), - values(var.node_pools_labels["all"]), - keys(var.node_pools_labels[var.node_pools[count.index]["name"]]), - values(var.node_pools_labels[var.node_pools[count.index]["name"]]) + keys(local.node_pools_labels["all"]), + values(local.node_pools_labels["all"]), + keys(local.node_pools_labels[var.node_pools[count.index]["name"]]), + values(local.node_pools_labels[var.node_pools[count.index]["name"]]) ) ) ) @@ -247,10 +247,10 @@ resource "random_id" "name" { metadata = join(",", sort( concat( - keys(var.node_pools_metadata["all"]), - values(var.node_pools_metadata["all"]), - keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]), - values(var.node_pools_metadata[var.node_pools[count.index]["name"]]) + keys(local.node_pools_metadata["all"]), + values(local.node_pools_metadata["all"]), + keys(local.node_pools_metadata[var.node_pools[count.index]["name"]]), + values(local.node_pools_metadata[var.node_pools[count.index]["name"]]) ) ) ) @@ -259,8 +259,8 @@ resource "random_id" "name" { oauth_scopes = join(",", sort( concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] ) ) ) @@ -269,8 +269,8 @@ resource "random_id" "name" { tags = join(",", sort( concat( - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]] + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]] ) ) ) @@ -285,8 +285,19 @@ resource "google_container_node_pool" "pools" { project = var.project_id location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified +<<<<<<< HEAD +<<<<<<< HEAD node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null - cluster = google_container_cluster.primary.name + cluster = google_container_cluster.primary.name +======= + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations +======= + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null +>>>>>>> Run autogen + + cluster = google_container_cluster.primary.name + +>>>>>>> Fix autogen code formatting issues which were inconsistent with what terraform format does to the generated code version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -320,24 +331,24 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -346,10 +357,10 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -364,8 +375,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -396,7 +407,8 @@ resource "google_container_node_pool" "pools" { } lifecycle { - ignore_changes = [initial_node_count] + ignore_changes = [initial_node_count] + create_before_destroy = true } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 5b235ce00f..bc466df2b2 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -132,6 +132,7 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d04ed5ac1f..65335a674c 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -203,6 +207,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -352,6 +357,7 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) + default = [{ state = "DECRYPTED" key_name = "" @@ -370,6 +376,7 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ "enabled" = false }] diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf new file mode 100644 index 0000000000..9a75764306 --- /dev/null +++ b/modules/beta-private-cluster-update-variant/variables_defaults.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 95b78225e7..afba3457c9 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -213,8 +213,19 @@ resource "google_container_node_pool" "pools" { project = var.project_id location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified +<<<<<<< HEAD +<<<<<<< HEAD node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null - cluster = google_container_cluster.primary.name + cluster = google_container_cluster.primary.name +======= + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations +======= + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null +>>>>>>> Run autogen + + cluster = google_container_cluster.primary.name + +>>>>>>> Fix autogen code formatting issues which were inconsistent with what terraform format does to the generated code version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -248,24 +259,24 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -274,10 +285,10 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -292,8 +303,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -325,6 +336,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 5b235ce00f..bc466df2b2 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -132,6 +132,7 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d04ed5ac1f..65335a674c 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -203,6 +207,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -352,6 +357,7 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) + default = [{ state = "DECRYPTED" key_name = "" @@ -370,6 +376,7 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ "enabled" = false }] diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf new file mode 100644 index 0000000000..9a75764306 --- /dev/null +++ b/modules/beta-private-cluster/variables_defaults.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 638d65af9d..e59e94aac7 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -208,8 +208,19 @@ resource "google_container_node_pool" "pools" { project = var.project_id location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified +<<<<<<< HEAD +<<<<<<< HEAD node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null - cluster = google_container_cluster.primary.name + cluster = google_container_cluster.primary.name +======= + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations +======= + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null +>>>>>>> Run autogen + + cluster = google_container_cluster.primary.name + +>>>>>>> Fix autogen code formatting issues which were inconsistent with what terraform format does to the generated code version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -243,24 +254,24 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -269,10 +280,10 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -287,8 +298,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -320,6 +331,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9668b6f1ea..dbc346ca0c 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -132,6 +132,7 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index a1057f1843..b875aea14f 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -203,6 +207,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -328,6 +333,7 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) + default = [{ state = "DECRYPTED" key_name = "" @@ -346,6 +352,7 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ "enabled" = false }] diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf new file mode 100644 index 0000000000..9a75764306 --- /dev/null +++ b/modules/beta-public-cluster/variables_defaults.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index af422abb2d..eebb4cd2ab 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -159,10 +159,10 @@ resource "random_id" "name" { labels = join(",", sort( concat( - keys(var.node_pools_labels["all"]), - values(var.node_pools_labels["all"]), - keys(var.node_pools_labels[var.node_pools[count.index]["name"]]), - values(var.node_pools_labels[var.node_pools[count.index]["name"]]) + keys(local.node_pools_labels["all"]), + values(local.node_pools_labels["all"]), + keys(local.node_pools_labels[var.node_pools[count.index]["name"]]), + values(local.node_pools_labels[var.node_pools[count.index]["name"]]) ) ) ) @@ -171,10 +171,10 @@ resource "random_id" "name" { metadata = join(",", sort( concat( - keys(var.node_pools_metadata["all"]), - values(var.node_pools_metadata["all"]), - keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]), - values(var.node_pools_metadata[var.node_pools[count.index]["name"]]) + keys(local.node_pools_metadata["all"]), + values(local.node_pools_metadata["all"]), + keys(local.node_pools_metadata[var.node_pools[count.index]["name"]]), + values(local.node_pools_metadata[var.node_pools[count.index]["name"]]) ) ) ) @@ -183,8 +183,8 @@ resource "random_id" "name" { oauth_scopes = join(",", sort( concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] ) ) ) @@ -193,8 +193,8 @@ resource "random_id" "name" { tags = join(",", sort( concat( - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]] + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]] ) ) ) @@ -208,7 +208,9 @@ resource "google_container_node_pool" "pools" { name = random_id.name.*.hex[count.index] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -241,25 +243,25 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -274,8 +276,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -290,7 +292,8 @@ resource "google_container_node_pool" "pools" { } lifecycle { - ignore_changes = [initial_node_count] + ignore_changes = [initial_node_count] + create_before_destroy = true } diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 508a4f1b96..52ad93f973 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf new file mode 100644 index 0000000000..145685f074 --- /dev/null +++ b/modules/private-cluster-update-variant/variables_defaults.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 4567530763..94f3637919 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -136,7 +136,9 @@ resource "google_container_node_pool" "pools" { name = var.node_pools[count.index]["name"] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -169,25 +171,25 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -202,8 +204,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -219,6 +221,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 508a4f1b96..52ad93f973 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf new file mode 100644 index 0000000000..145685f074 --- /dev/null +++ b/modules/private-cluster/variables_defaults.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/test/integration/deploy_service/controls/kubectl.rb b/test/integration/deploy_service/controls/kubectl.rb index 2d4a473d2c..67921b1298 100644 --- a/test/integration/deploy_service/controls/kubectl.rb +++ b/test/integration/deploy_service/controls/kubectl.rb @@ -58,6 +58,12 @@ it "is reachable" do expect { + 10.times do + unless host(service_load_balancer_ip, port: 8080, protocol: 'tcp').reachable? + puts "Nginx is not reachable, retrying.." + sleep 10 + end + end RestClient.get(service_load_balancer_address) }.to_not raise_exception end diff --git a/variables.tf b/variables.tf index 58cf1f4685..5b89c692ec 100644 --- a/variables.tf +++ b/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] diff --git a/variables_defaults.tf b/variables_defaults.tf new file mode 100644 index 0000000000..145685f074 --- /dev/null +++ b/variables_defaults.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +}