From 16bdd6e6310ae248991462494f50876b99a36bbe Mon Sep 17 00:00:00 2001 From: Devan Patel Date: Thu, 23 Apr 2020 15:59:10 +0100 Subject: [PATCH] feat: Add support for setting firewall rules (#470) * Squash Commits Signed-off-by: Dev * Fix example Signed-off-by: Dev * Rename var + update README Signed-off-by: Dev * Set to false as default Signed-off-by: Dev * Enable firewall support in shared_vpc example Signed-off-by: Dev * Remove network datasource and make subnetwork conditional on firewall Signed-off-by: Dev * Fix attribute error Signed-off-by: Dev --- README.md | 4 + autogen/main/README.md | 1 + autogen/main/cluster.tf.tmpl | 17 ++-- autogen/main/firewall.tf.tmpl | 84 +++++++++++++++++++ autogen/main/main.tf.tmpl | 32 ++++--- autogen/main/networks.tf | 26 ++++++ autogen/main/variables.tf.tmpl | 21 ++++- autogen/safer-cluster/variables.tf.tmpl | 2 +- cluster.tf | 5 +- .../private_zonal_with_networking/main.tf | 8 +- examples/shared_vpc/main.tf | 24 +++--- firewall.tf | 84 +++++++++++++++++++ main.tf | 7 +- .../README.md | 4 + .../cluster.tf | 7 +- .../firewall.tf | 84 +++++++++++++++++++ .../main.tf | 9 +- .../networks.tf | 26 ++++++ .../variables.tf | 19 +++++ modules/beta-private-cluster/README.md | 4 + modules/beta-private-cluster/cluster.tf | 7 +- modules/beta-private-cluster/firewall.tf | 84 +++++++++++++++++++ modules/beta-private-cluster/main.tf | 9 +- modules/beta-private-cluster/networks.tf | 26 ++++++ modules/beta-private-cluster/variables.tf | 19 +++++ modules/beta-public-cluster/README.md | 4 + modules/beta-public-cluster/cluster.tf | 7 +- modules/beta-public-cluster/firewall.tf | 84 +++++++++++++++++++ modules/beta-public-cluster/main.tf | 7 +- modules/beta-public-cluster/networks.tf | 26 ++++++ modules/beta-public-cluster/variables.tf | 19 +++++ .../private-cluster-update-variant/README.md | 4 + .../private-cluster-update-variant/cluster.tf | 5 +- .../firewall.tf | 84 +++++++++++++++++++ .../private-cluster-update-variant/main.tf | 9 +- .../networks.tf | 26 ++++++ .../variables.tf | 19 +++++ modules/private-cluster/README.md | 4 + modules/private-cluster/cluster.tf | 5 +- modules/private-cluster/firewall.tf | 84 +++++++++++++++++++ modules/private-cluster/main.tf | 9 +- modules/private-cluster/networks.tf | 26 ++++++ modules/private-cluster/variables.tf | 19 +++++ networks.tf | 8 +- test/setup/iam.tf | 2 +- variables.tf | 19 +++++ 46 files changed, 1010 insertions(+), 73 deletions(-) create mode 100644 autogen/main/firewall.tf.tmpl create mode 100644 autogen/main/networks.tf create mode 100644 firewall.tf create mode 100644 modules/beta-private-cluster-update-variant/firewall.tf create mode 100644 modules/beta-private-cluster-update-variant/networks.tf create mode 100644 modules/beta-private-cluster/firewall.tf create mode 100644 modules/beta-private-cluster/networks.tf create mode 100644 modules/beta-public-cluster/firewall.tf create mode 100644 modules/beta-public-cluster/networks.tf create mode 100644 modules/private-cluster-update-variant/firewall.tf create mode 100644 modules/private-cluster-update-variant/networks.tf create mode 100644 modules/private-cluster/firewall.tf create mode 100644 modules/private-cluster/networks.tf diff --git a/README.md b/README.md index 6c4ef7f5a..b4ebb2cf7 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no | @@ -109,6 +110,8 @@ Then perform the following commands on the root folder: | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | | description | The description of the cluster | string | `""` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | +| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | @@ -219,6 +222,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/autogen/main/README.md b/autogen/main/README.md index e7ea96338..dec676c1f 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -240,6 +240,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 8cc0bda8b..841b8cf08 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -82,6 +82,7 @@ resource "google_container_cluster" "primary" { {% endif %} default_max_pods_per_node = var.default_max_pods_per_node + {% if beta_cluster %} enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility @@ -159,7 +160,7 @@ resource "google_container_cluster" "primary" { } dns_cache_config { - enabled = var.dns_cache + enabled = var.dns_cache } {% endif %} } @@ -171,16 +172,16 @@ resource "google_container_cluster" "primary" { maintenance_policy { {% if beta_cluster %} - dynamic "recurring_window"{ + dynamic "recurring_window" { for_each = local.cluster_maintenance_window_is_recurring content { start_time = var.maintenance_start_time - end_time = var.maintenance_end_time + end_time = var.maintenance_end_time recurrence = var.maintenance_recurrence } } - dynamic "daily_maintenance_window"{ + dynamic "daily_maintenance_window" { for_each = local.cluster_maintenance_window_is_daily content { start_time = var.maintenance_start_time @@ -352,7 +353,7 @@ resource "google_container_node_pool" "pools" { {% endif %} for_each = local.node_pools {% if update_variant %} - name = {for k, v in random_id.name : k => v.hex}[each.key] + name = { for k, v in random_id.name : k => v.hex }[each.key] {% else %} name = each.key {% endif %} @@ -396,7 +397,7 @@ resource "google_container_node_pool" "pools" { {% if beta_cluster %} upgrade_settings { - max_surge = lookup(each.value, "max_surge", 1) + max_surge = lookup(each.value, "max_surge", 1) max_unavailable = lookup(each.value, "max_unavailable", 0) } {% endif %} @@ -433,8 +434,8 @@ resource "google_container_node_pool" "pools" { } {% endif %} tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl new file mode 100644 index 000000000..8e53b6eee --- /dev/null +++ b/autogen/main/firewall.tf.tmpl @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{{ autogeneration_note }} + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 04773f187..491d1f7ba 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -51,17 +51,17 @@ locals { node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) {% if beta_cluster %} - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ - resource_type = "cpu" - minimum = var.cluster_autoscaling.min_cpu_cores - maximum = var.cluster_autoscaling.max_cpu_cores + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores }, { - resource_type = "memory" - minimum = var.cluster_autoscaling.min_memory_gb - maximum = var.cluster_autoscaling.max_memory_gb - }] : [] + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] {% endif %} @@ -74,6 +74,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -103,10 +106,12 @@ locals { cluster_output_zones = local.cluster_output_regional_zones {% if private_cluster %} - cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint - cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint + cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint_for_nodes = var.master_ipv4_cidr_block {% else %} - cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint_for_nodes = "${google_container_cluster.primary.endpoint}/32" {% endif %} cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) @@ -144,6 +149,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version @@ -164,14 +170,14 @@ locals { cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled cluster_workload_identity_config = var.identity_namespace == null ? [] : var.identity_namespace == "enabled" ? [{ - identity_namespace = "${var.project_id}.svc.id.goog"}] : [{identity_namespace = var.identity_namespace + identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace }] # /BETA features {% endif %} {% if beta_cluster %} cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] - cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] + cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] {% endif %} } diff --git a/autogen/main/networks.tf b/autogen/main/networks.tf new file mode 100644 index 000000000..9a4726987 --- /dev/null +++ b/autogen/main/networks.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{{ autogeneration_note }} + +data "google_compute_subnetwork" "gke_subnetwork" { + provider = google + + count = var.add_cluster_firewall_rules ? 1 : 0 + name = var.subnetwork + region = local.region + project = local.network_project_id +} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index d36d063da..4525819e5 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -420,7 +420,7 @@ variable "enable_binary_authorization" { } variable "pod_security_policy_config" { - type = list(object({ enabled = bool })) + type = list(object({ enabled = bool })) description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = [{ @@ -482,3 +482,22 @@ variable "enable_shielded_nodes" { default = true } {% endif %} + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +} diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 9e1a54dbc..1d82c13b9 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -317,7 +317,7 @@ variable "skip_provisioners" { } variable "pod_security_policy_config" { - type = list(object({ enabled = bool })) + type = list(object({ enabled = bool })) description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." default = [{ diff --git a/cluster.tf b/cluster.tf index 439827914..b9c937e18 100644 --- a/cluster.tf +++ b/cluster.tf @@ -51,6 +51,7 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -182,8 +183,8 @@ resource "google_container_node_pool" "pools" { }, ) tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index 18af8bbd7..006dee61a 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -58,12 +58,8 @@ module "gke" { region = var.region zones = slice(var.zones, 0, 1) - // This craziness gets a plain network name from the reference link which is the - // only way to force cluster creation to wait on network creation without a - // depends_on link. Tests use terraform 0.12.6, which does not have regex or regexall - network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0] - - subnetwork = data.google_compute_subnetwork.subnetwork.name + network = module.gcp-network.network_name + subnetwork = module.gcp-network.subnets_names[0] ip_range_pods = var.ip_range_pods_name ip_range_services = var.ip_range_services_name create_service_account = true diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf index a544aa62f..8c554fddb 100644 --- a/examples/shared_vpc/main.tf +++ b/examples/shared_vpc/main.tf @@ -24,17 +24,19 @@ provider "google" { } module "gke" { - source = "../../" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - region = var.region - network = var.network - network_project_id = var.network_project_id - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account + source = "../../" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + region = var.region + network = var.network + network_project_id = var.network_project_id + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + add_cluster_firewall_rules = true + firewall_inbound_ports = ["9443", "15017"] } data "google_client_config" "default" { diff --git a/firewall.tf b/firewall.tf new file mode 100644 index 000000000..0d0fa0718 --- /dev/null +++ b/firewall.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/main.tf b/main.tf index 579e2837e..a27a97c0a 100644 --- a/main.tf +++ b/main.tf @@ -56,6 +56,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -70,7 +73,8 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint_for_nodes = "${google_container_cluster.primary.endpoint}/32" cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version @@ -98,6 +102,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index e9fa0122c..7940ba2a9 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -156,6 +156,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | @@ -178,6 +179,8 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | +| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | @@ -319,6 +322,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 66f2024c4..53b22c716 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -69,7 +69,8 @@ resource "google_container_cluster" "primary" { } } - default_max_pods_per_node = var.default_max_pods_per_node + default_max_pods_per_node = var.default_max_pods_per_node + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility enable_shielded_nodes = var.enable_shielded_nodes @@ -389,8 +390,8 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf new file mode 100644 index 000000000..0d0fa0718 --- /dev/null +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 8fd0633f3..6127c323f 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -68,6 +68,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -94,8 +97,9 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint - cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint + cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint_for_nodes = var.master_ipv4_cidr_block cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version @@ -130,6 +134,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf new file mode 100644 index 000000000..295263c29 --- /dev/null +++ b/modules/beta-private-cluster-update-variant/networks.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +data "google_compute_subnetwork" "gke_subnetwork" { + provider = google + + count = var.add_cluster_firewall_rules ? 1 : 0 + name = var.subnetwork + region = local.region + project = local.network_project_id +} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index ade9eb208..aff8f1a62 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -474,3 +474,22 @@ variable "enable_shielded_nodes" { description = "Enable Shielded Nodes features on all nodes in this cluster" default = true } + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +} diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 0b9d9b1dc..a73e37ec2 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -134,6 +134,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | @@ -156,6 +157,8 @@ Then perform the following commands on the root folder: | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | +| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | @@ -297,6 +300,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 403d9eaae..98c3a7362 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -69,7 +69,8 @@ resource "google_container_cluster" "primary" { } } - default_max_pods_per_node = var.default_max_pods_per_node + default_max_pods_per_node = var.default_max_pods_per_node + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility enable_shielded_nodes = var.enable_shielded_nodes @@ -317,8 +318,8 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf new file mode 100644 index 000000000..0d0fa0718 --- /dev/null +++ b/modules/beta-private-cluster/firewall.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 8fd0633f3..6127c323f 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -68,6 +68,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -94,8 +97,9 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint - cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint + cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint_for_nodes = var.master_ipv4_cidr_block cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version @@ -130,6 +134,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version diff --git a/modules/beta-private-cluster/networks.tf b/modules/beta-private-cluster/networks.tf new file mode 100644 index 000000000..295263c29 --- /dev/null +++ b/modules/beta-private-cluster/networks.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +data "google_compute_subnetwork" "gke_subnetwork" { + provider = google + + count = var.add_cluster_firewall_rules ? 1 : 0 + name = var.subnetwork + region = local.region + project = local.network_project_id +} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index ade9eb208..aff8f1a62 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -474,3 +474,22 @@ variable "enable_shielded_nodes" { description = "Enable Shielded Nodes features on all nodes in this cluster" default = true } + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +} diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 8eabf9981..3a4ad3795 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -116,6 +116,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | @@ -135,6 +136,8 @@ Then perform the following commands on the root folder: | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | bool | `"false"` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"true"` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | +| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | @@ -273,6 +276,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 26651b917..3d8678a05 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -69,7 +69,8 @@ resource "google_container_cluster" "primary" { } } - default_max_pods_per_node = var.default_max_pods_per_node + default_max_pods_per_node = var.default_max_pods_per_node + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility enable_shielded_nodes = var.enable_shielded_nodes @@ -304,8 +305,8 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf new file mode 100644 index 000000000..0d0fa0718 --- /dev/null +++ b/modules/beta-public-cluster/firewall.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 2512552bd..bbdf9192a 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -68,6 +68,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -94,7 +97,8 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint = google_container_cluster.primary.endpoint + cluster_endpoint_for_nodes = "${google_container_cluster.primary.endpoint}/32" cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version @@ -129,6 +133,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version diff --git a/modules/beta-public-cluster/networks.tf b/modules/beta-public-cluster/networks.tf new file mode 100644 index 000000000..295263c29 --- /dev/null +++ b/modules/beta-public-cluster/networks.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +data "google_compute_subnetwork" "gke_subnetwork" { + provider = google + + count = var.add_cluster_firewall_rules ? 1 : 0 + name = var.subnetwork + region = local.region + project = local.network_project_id +} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index df9dbf31a..2bb84a705 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -450,3 +450,22 @@ variable "enable_shielded_nodes" { description = "Enable Shielded Nodes features on all nodes in this cluster" default = true } + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +} diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index b56d8dff7..9fb0dfe5b 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -140,6 +140,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no | @@ -152,6 +153,8 @@ Then perform the following commands on the root folder: | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | +| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | @@ -265,6 +268,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 04b3a7892..7128c2de4 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -51,6 +51,7 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -267,8 +268,8 @@ resource "google_container_node_pool" "pools" { }, ) tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf new file mode 100644 index 000000000..0d0fa0718 --- /dev/null +++ b/modules/private-cluster-update-variant/firewall.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index f9b57ff9b..87bd0f47a 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -56,6 +56,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -70,8 +73,9 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint - cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint + cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint_for_nodes = var.master_ipv4_cidr_block cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version @@ -99,6 +103,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf new file mode 100644 index 000000000..295263c29 --- /dev/null +++ b/modules/private-cluster-update-variant/networks.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +data "google_compute_subnetwork" "gke_subnetwork" { + provider = google + + count = var.add_cluster_firewall_rules ? 1 : 0 + name = var.subnetwork + region = local.region + project = local.network_project_id +} diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index fc5bf996d..7ad0f86ba 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -325,3 +325,22 @@ variable "master_ipv4_cidr_block" { description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" } + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +} diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 087ca5e19..0c23d6109 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -118,6 +118,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no | @@ -130,6 +131,8 @@ Then perform the following commands on the root folder: | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `` | no | +| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | @@ -243,6 +246,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog In order to execute this module you must have a Service Account with the following project roles: - roles/compute.viewer +- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`) - roles/container.clusterAdmin - roles/container.developer - roles/iam.serviceAccountAdmin diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index d53b5aca8..75dbeb36f 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -51,6 +51,7 @@ resource "google_container_cluster" "primary" { default_max_pods_per_node = var.default_max_pods_per_node + dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -195,8 +196,8 @@ resource "google_container_node_pool" "pools" { }, ) tags = concat( - lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [], local.node_pools_tags["all"], local.node_pools_tags[each.value["name"]], ) diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf new file mode 100644 index 000000000..0d0fa0718 --- /dev/null +++ b/modules/private-cluster/firewall.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + + +/****************************************** + Match the gke---all INGRESS + firewall rule created by GKE but for EGRESS + + Required for clusters when VPCs enforce + a default-deny egress rule + *****************************************/ +resource "google_compute_firewall" "intra_egress" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress" + description = "Managed by terraform gke module: Allow pods to communicate with each other and the master" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "EGRESS" + + target_tags = [local.cluster_network_tag] + destination_ranges = [ + local.cluster_endpoint_for_nodes, + local.cluster_subnet_cidr, + local.cluster_alias_ranges_cidr[var.ip_range_pods], + ] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + depends_on = [ + google_container_cluster.primary, + ] +} + + +/****************************************** + Allow GKE master to hit non 443 ports for + Webhooks/Admission Controllers + + https://github.com/kubernetes/kubernetes/issues/79739 + *****************************************/ +resource "google_compute_firewall" "master_webhooks" { + count = var.add_cluster_firewall_rules ? 1 : 0 + name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks" + description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks" + project = local.network_project_id + network = var.network + priority = var.firewall_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = var.firewall_inbound_ports + } + + depends_on = [ + google_container_cluster.primary, + ] + +} diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index f9b57ff9b..87bd0f47a 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -56,6 +56,9 @@ locals { // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. default_auto_upgrade = var.regional ? true : false + cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null + cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider @@ -70,8 +73,9 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint - cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint + cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null + cluster_endpoint_for_nodes = var.master_ipv4_cidr_block cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version @@ -99,6 +103,7 @@ locals { cluster_zones = sort(local.cluster_output_zones) cluster_name = local.cluster_output_name + cluster_network_tag = "gke-${var.name}" cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version cluster_min_master_version = local.cluster_output_min_master_version diff --git a/modules/private-cluster/networks.tf b/modules/private-cluster/networks.tf new file mode 100644 index 000000000..295263c29 --- /dev/null +++ b/modules/private-cluster/networks.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen/main + +data "google_compute_subnetwork" "gke_subnetwork" { + provider = google + + count = var.add_cluster_firewall_rules ? 1 : 0 + name = var.subnetwork + region = local.region + project = local.network_project_id +} diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index fc5bf996d..7ad0f86ba 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -325,3 +325,22 @@ variable "master_ipv4_cidr_block" { description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" } + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +} diff --git a/networks.tf b/networks.tf index 9281d0632..295263c29 100644 --- a/networks.tf +++ b/networks.tf @@ -16,16 +16,10 @@ // This file was automatically generated from a template in ./autogen/main -data "google_compute_network" "gke_network" { - provider = google - - name = var.network - project = local.network_project_id -} - data "google_compute_subnetwork" "gke_subnetwork" { provider = google + count = var.add_cluster_firewall_rules ? 1 : 0 name = var.subnetwork region = local.region project = local.network_project_id diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 8685b9af5..4615d0844 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -19,12 +19,12 @@ locals { "roles/cloudkms.admin", "roles/cloudkms.cryptoKeyEncrypterDecrypter", "roles/compute.networkAdmin", + "roles/compute.securityAdmin", "roles/container.admin", "roles/container.clusterAdmin", "roles/container.developer", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", - "roles/compute.networkAdmin", "roles/compute.viewer", "roles/resourcemanager.projectIamAdmin", "roles/composer.worker" diff --git a/variables.tf b/variables.tf index fd039cc94..5f8da02b5 100644 --- a/variables.tf +++ b/variables.tf @@ -301,3 +301,22 @@ variable "default_max_pods_per_node" { description = "The maximum number of pods to schedule per node" default = 110 } + + +variable "add_cluster_firewall_rules" { + type = bool + description = "Create additional firewall rules" + default = false +} + +variable "firewall_priority" { + type = number + description = "Priority rule for firewall rules" + default = 1000 +} + +variable "firewall_inbound_ports" { + type = list(string) + description = "List of TCP ports for admission/webhook controllers" + default = ["8443", "9443", "15017"] +}