feat: Add support for setting firewall rules (#470)

* Squash Commits Signed-off-by: Dev <Dev25@users.noreply.github.com> * Fix example Signed-off-by: Dev <Dev25@users.noreply.github.com> * Rename var + update README Signed-off-by: Dev <Dev25@users.noreply.github.com> * Set to false as default Signed-off-by: Dev <Dev25@users.noreply.github.com> * Enable firewall support in shared_vpc example Signed-off-by: Dev <Dev25@users.noreply.github.com> * Remove network datasource and make subnetwork conditional on firewall Signed-off-by: Dev <Dev25@users.noreply.github.com> * Fix attribute error Signed-off-by: Dev <Dev25@users.noreply.github.com>
terraform-google-modules · Apr 23, 2020 · 16bdd6e · 16bdd6e
1 parent 5791ac1
commit 16bdd6e
Show file tree

Hide file tree

Showing 46 changed files with 1,010 additions and 73 deletions.
diff --git a/README.md b/README.md
@@ -100,6 +100,7 @@ Then perform the following commands on the root folder:
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| add\_cluster\_firewall\_rules | Create additional firewall rules | bool | `"false"` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `"null"` | no |
@@ -109,6 +110,8 @@ Then perform the following commands on the root folder:
 | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no |
 | description | The description of the cluster | string | `""` | no |
 | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no |
+| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | list(string) | `<list>` | no |
+| firewall\_priority | Priority rule for firewall rules | number | `"1000"` | no |
 | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no |
 | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no |
 | http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no |
@@ -219,6 +222,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 In order to execute this module you must have a Service Account with the
 following project roles:
 - roles/compute.viewer
+- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`)
 - roles/container.clusterAdmin
 - roles/container.developer
 - roles/iam.serviceAccountAdmin

diff --git a/autogen/main/README.md b/autogen/main/README.md
@@ -240,6 +240,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 In order to execute this module you must have a Service Account with the
 following project roles:
 - roles/compute.viewer
+- roles/compute.securityAdmin (only required if `add_cluster_firewall_rules` is set to `true`)
 - roles/container.clusterAdmin
 - roles/container.developer
 - roles/iam.serviceAccountAdmin

diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -82,6 +82,7 @@ resource "google_container_cluster" "primary" {
 {% endif %}
 
   default_max_pods_per_node = var.default_max_pods_per_node
+
 {% if beta_cluster %}
   enable_binary_authorization = var.enable_binary_authorization
   enable_intranode_visibility = var.enable_intranode_visibility
@@ -159,7 +160,7 @@ resource "google_container_cluster" "primary" {
     }
 
     dns_cache_config {
-      enabled   = var.dns_cache
+      enabled = var.dns_cache
     }
     {% endif %}
   }
@@ -171,16 +172,16 @@ resource "google_container_cluster" "primary" {
 
   maintenance_policy {
   {% if beta_cluster %}
-    dynamic "recurring_window"{
+    dynamic "recurring_window" {
       for_each = local.cluster_maintenance_window_is_recurring
       content {
         start_time = var.maintenance_start_time
-        end_time = var.maintenance_end_time
+        end_time   = var.maintenance_end_time
         recurrence = var.maintenance_recurrence
       }
     }
 
-    dynamic "daily_maintenance_window"{
+    dynamic "daily_maintenance_window" {
       for_each = local.cluster_maintenance_window_is_daily
       content {
         start_time = var.maintenance_start_time
@@ -352,7 +353,7 @@ resource "google_container_node_pool" "pools" {
   {% endif %}
   for_each = local.node_pools
   {% if update_variant %}
-  name     = {for k, v in random_id.name : k => v.hex}[each.key]
+  name     = { for k, v in random_id.name : k => v.hex }[each.key]
   {% else %}
   name     = each.key
   {% endif %}
@@ -396,7 +397,7 @@ resource "google_container_node_pool" "pools" {
 
   {% if beta_cluster %}
   upgrade_settings {
-    max_surge = lookup(each.value, "max_surge", 1)
+    max_surge       = lookup(each.value, "max_surge", 1)
     max_unavailable = lookup(each.value, "max_unavailable", 0)
   }
   {% endif %}
@@ -433,8 +434,8 @@ resource "google_container_node_pool" "pools" {
     }
     {% endif %}
     tags = concat(
-      lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
-      lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [],
+      lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [],
+      lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [],
       local.node_pools_tags["all"],
       local.node_pools_tags[each.value["name"]],
     )

diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{{ autogeneration_note }}
+
+
+/******************************************
+  Match the gke-<CLUSTER>-<ID>-all INGRESS
+  firewall rule created by GKE but for EGRESS
+
+  Required for clusters when VPCs enforce
+  a default-deny egress rule
+ *****************************************/
+resource "google_compute_firewall" "intra_egress" {
+  count       = var.add_cluster_firewall_rules ? 1 : 0
+  name        = "gke-${substr(var.name, 0, min(25, length(var.name)))}-intra-cluster-egress"
+  description = "Managed by terraform gke module: Allow pods to communicate with each other and the master"
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.firewall_priority
+  direction   = "EGRESS"
+
+  target_tags = [local.cluster_network_tag]
+  destination_ranges = [
+    local.cluster_endpoint_for_nodes,
+    local.cluster_subnet_cidr,
+    local.cluster_alias_ranges_cidr[var.ip_range_pods],
+  ]
+
+  # Allow all possible protocols
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+  allow { protocol = "sctp" }
+  allow { protocol = "esp" }
+  allow { protocol = "ah" }
+
+  depends_on = [
+    google_container_cluster.primary,
+  ]
+}
+
+
+/******************************************
+  Allow GKE master to hit non 443 ports for
+  Webhooks/Admission Controllers
+
+  https://github.com/kubernetes/kubernetes/issues/79739
+ *****************************************/
+resource "google_compute_firewall" "master_webhooks" {
+  count       = var.add_cluster_firewall_rules ? 1 : 0
+  name        = "gke-${substr(var.name, 0, min(25, length(var.name)))}-webhooks"
+  description = "Managed by terraform gke module: Allow master to hit pods for admission controllers/webhooks"
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.firewall_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_endpoint_for_nodes]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "tcp"
+    ports    = var.firewall_inbound_ports
+  }
+
+  depends_on = [
+    google_container_cluster.primary,
+  ]
+
+}
diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl
@@ -51,17 +51,17 @@ locals {
   node_pools      = zipmap(local.node_pool_names, tolist(toset(var.node_pools)))
 
 {% if beta_cluster %}
-  release_channel         = var.release_channel != null ? [{ channel : var.release_channel }] : []
+  release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : []
 
   autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{
-      resource_type = "cpu"
-      minimum       = var.cluster_autoscaling.min_cpu_cores
-      maximum       = var.cluster_autoscaling.max_cpu_cores
+    resource_type = "cpu"
+    minimum       = var.cluster_autoscaling.min_cpu_cores
+    maximum       = var.cluster_autoscaling.max_cpu_cores
     }, {
-      resource_type = "memory"
-      minimum       = var.cluster_autoscaling.min_memory_gb
-      maximum       = var.cluster_autoscaling.max_memory_gb
-    }] : []
+    resource_type = "memory"
+    minimum       = var.cluster_autoscaling.min_memory_gb
+    maximum       = var.cluster_autoscaling.max_memory_gb
+  }] : []
 
 {% endif %}
 
@@ -74,6 +74,9 @@ locals {
   // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous.
   default_auto_upgrade = var.regional ? true : false
 
+  cluster_subnet_cidr       = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null
+  cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {}
+
   cluster_network_policy = var.network_policy ? [{
     enabled  = true
     provider = var.network_policy_provider
@@ -103,10 +106,12 @@ locals {
   cluster_output_zones          = local.cluster_output_regional_zones
 
 {% if private_cluster %}
-  cluster_endpoint     = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint
-  cluster_peering_name = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null
+  cluster_endpoint           = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? (var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint) : google_container_cluster.primary.endpoint
+  cluster_peering_name       = (var.enable_private_nodes && length(google_container_cluster.primary.private_cluster_config) > 0) ? google_container_cluster.primary.private_cluster_config.0.peering_name : null
+  cluster_endpoint_for_nodes = var.master_ipv4_cidr_block
 {% else %}
-  cluster_endpoint = google_container_cluster.primary.endpoint
+  cluster_endpoint           = google_container_cluster.primary.endpoint
+  cluster_endpoint_for_nodes = "${google_container_cluster.primary.endpoint}/32"
 {% endif %}
 
   cluster_output_master_auth                        = concat(google_container_cluster.primary.*.master_auth, [])
@@ -144,6 +149,7 @@ locals {
   cluster_zones    = sort(local.cluster_output_zones)
 
   cluster_name                               = local.cluster_output_name
+  cluster_network_tag                        = "gke-${var.name}"
   cluster_ca_certificate                     = local.cluster_master_auth_map["cluster_ca_certificate"]
   cluster_master_version                     = local.cluster_output_master_version
   cluster_min_master_version                 = local.cluster_output_min_master_version
@@ -164,14 +170,14 @@ locals {
   cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled
 
   cluster_workload_identity_config = var.identity_namespace == null ? [] : var.identity_namespace == "enabled" ? [{
-  identity_namespace = "${var.project_id}.svc.id.goog"}] : [{identity_namespace = var.identity_namespace
+    identity_namespace = "${var.project_id}.svc.id.goog" }] : [{ identity_namespace = var.identity_namespace
   }]
   # /BETA features
 {% endif %}
 
 {% if beta_cluster %}
   cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : []
-  cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]
+  cluster_maintenance_window_is_daily     = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1]
 {% endif %}
 }
 

diff --git a/autogen/main/networks.tf b/autogen/main/networks.tf
@@ -0,0 +1,26 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{{ autogeneration_note }}
+
+data "google_compute_subnetwork" "gke_subnetwork" {
+  provider = google
+
+  count   = var.add_cluster_firewall_rules ? 1 : 0
+  name    = var.subnetwork
+  region  = local.region
+  project = local.network_project_id
+}
diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -420,7 +420,7 @@ variable "enable_binary_authorization" {
 }
 
 variable "pod_security_policy_config" {
-  type = list(object({ enabled = bool }))
+  type        = list(object({ enabled = bool }))
   description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created."
 
   default = [{
@@ -482,3 +482,22 @@ variable "enable_shielded_nodes" {
   default     = true
 }
 {% endif %}
+
+
+variable "add_cluster_firewall_rules" {
+  type        = bool
+  description = "Create additional firewall rules"
+  default     = false
+}
+
+variable "firewall_priority" {
+  type        = number
+  description = "Priority rule for firewall rules"
+  default     = 1000
+}
+
+variable "firewall_inbound_ports" {
+  type        = list(string)
+  description = "List of TCP ports for admission/webhook controllers"
+  default     = ["8443", "9443", "15017"]
+}
diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl
@@ -317,7 +317,7 @@ variable "skip_provisioners" {
 }
 
 variable "pod_security_policy_config" {
-  type = list(object({ enabled = bool }))
+  type        = list(object({ enabled = bool }))
   description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created."
 
   default = [{

diff --git a/cluster.tf b/cluster.tf
@@ -51,6 +51,7 @@ resource "google_container_cluster" "primary" {
 
 
   default_max_pods_per_node = var.default_max_pods_per_node
+
   dynamic "master_authorized_networks_config" {
     for_each = local.master_authorized_networks_config
     content {
@@ -182,8 +183,8 @@ resource "google_container_node_pool" "pools" {
       },
     )
     tags = concat(
-      lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [],
-      lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${each.value["name"]}"] : [],
+      lookup(local.node_pools_tags, "default_values", [true, true])[0] ? [local.cluster_network_tag] : [],
+      lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["${local.cluster_network_tag}-${each.value["name"]}"] : [],
       local.node_pools_tags["all"],
       local.node_pools_tags[each.value["name"]],
     )

diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf
@@ -58,12 +58,8 @@ module "gke" {
   region     = var.region
   zones      = slice(var.zones, 0, 1)
 
-  // This craziness gets a plain network name from the reference link which is the
-  // only way to force cluster creation to wait on network creation without a
-  // depends_on link.  Tests use terraform 0.12.6, which does not have regex or regexall
-  network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0]
-
-  subnetwork              = data.google_compute_subnetwork.subnetwork.name
+  network                 = module.gcp-network.network_name
+  subnetwork              = module.gcp-network.subnets_names[0]
   ip_range_pods           = var.ip_range_pods_name
   ip_range_services       = var.ip_range_services_name
   create_service_account  = true

diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf
@@ -24,17 +24,19 @@ provider "google" {
 }
 
 module "gke" {
-  source                 = "../../"
-  project_id             = var.project_id
-  name                   = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
-  region                 = var.region
-  network                = var.network
-  network_project_id     = var.network_project_id
-  subnetwork             = var.subnetwork
-  ip_range_pods          = var.ip_range_pods
-  ip_range_services      = var.ip_range_services
-  create_service_account = false
-  service_account        = var.compute_engine_service_account
+  source                     = "../../"
+  project_id                 = var.project_id
+  name                       = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  region                     = var.region
+  network                    = var.network
+  network_project_id         = var.network_project_id
+  subnetwork                 = var.subnetwork
+  ip_range_pods              = var.ip_range_pods
+  ip_range_services          = var.ip_range_services
+  create_service_account     = false
+  service_account            = var.compute_engine_service_account
+  add_cluster_firewall_rules = true
+  firewall_inbound_ports     = ["9443", "15017"]
 }
 
 data "google_client_config" "default" {