feat: Add support for creating "shadow" firewall rules for logging pu…

…rposes (#741)
terraform-google-modules · Feb 16, 2021 · 259dbfb · 259dbfb
1 parent f32dea7
commit 259dbfb
Show file tree

Hide file tree

Showing 23 changed files with 798 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -124,6 +124,7 @@ Then perform the following commands on the root folder:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no |
+| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no |
 | cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | <pre>object({<br>    enabled       = bool<br>    min_cpu_cores = number<br>    max_cpu_cores = number<br>    min_memory_gb = number<br>    max_memory_gb = number<br>  })</pre> | <pre>{<br>  "enabled": false,<br>  "max_cpu_cores": 0,<br>  "max_memory_gb": 0,<br>  "min_cpu_cores": 0,<br>  "min_memory_gb": 0<br>}</pre> | no |
@@ -181,6 +182,7 @@ Then perform the following commands on the root folder:
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no |
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
+| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes |

diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl
@@ -95,3 +95,89 @@ resource "google_compute_firewall" "master_webhooks" {
   {% endif %}
 
 }
+
+
+/******************************************
+  Create shadow firewall rules to capture the
+  traffic flow between the managed firewall rules
+ *****************************************/
+resource "google_compute_firewall" "shadow_allow_pods" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all"
+  description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
+  target_tags   = [local.cluster_network_tag]
+
+  # Allow all possible protocols
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+  allow { protocol = "sctp" }
+  allow { protocol = "esp" }
+  allow { protocol = "ah" }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_master" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_endpoint_for_nodes]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["10250", "443"]
+  }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_nodes" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms"
+  description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_subnet_cidr]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "icmp"
+  }
+
+  allow {
+    protocol = "udp"
+    ports    = ["1-65535"]
+  }
+
+  allow {
+    protocol = "tcp"
+    ports    = ["1-65535"]
+  }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -580,6 +580,18 @@ variable "gcloud_upgrade" {
   default     = false
 }
 
+variable "add_shadow_firewall_rules" {
+  type = bool
+  description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)."
+  default = false
+}
+
+variable "shadow_firewall_rules_priority" {
+  type = number
+  description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000."
+  default = 999
+}
+
 {% if beta_cluster %}
 variable "disable_default_snat" {
   type        = bool

diff --git a/firewall.tf b/firewall.tf
@@ -82,3 +82,89 @@ resource "google_compute_firewall" "master_webhooks" {
   ]
 
 }
+
+
+/******************************************
+  Create shadow firewall rules to capture the
+  traffic flow between the managed firewall rules
+ *****************************************/
+resource "google_compute_firewall" "shadow_allow_pods" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all"
+  description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
+  target_tags   = [local.cluster_network_tag]
+
+  # Allow all possible protocols
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+  allow { protocol = "sctp" }
+  allow { protocol = "esp" }
+  allow { protocol = "ah" }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_master" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_endpoint_for_nodes]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["10250", "443"]
+  }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_nodes" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms"
+  description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_subnet_cidr]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "icmp"
+  }
+
+  allow {
+    protocol = "udp"
+    ports    = ["1-65535"]
+  }
+
+  allow {
+    protocol = "tcp"
+    ports    = ["1-65535"]
+  }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md
@@ -155,6 +155,7 @@ Then perform the following commands on the root folder:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no |
+| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no |
 | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no |
@@ -237,6 +238,7 @@ Then perform the following commands on the root folder:
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
+| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes |

diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf
@@ -77,3 +77,89 @@ resource "google_compute_firewall" "master_webhooks" {
 
 
 }
+
+
+/******************************************
+  Create shadow firewall rules to capture the
+  traffic flow between the managed firewall rules
+ *****************************************/
+resource "google_compute_firewall" "shadow_allow_pods" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all"
+  description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]]
+  target_tags   = [local.cluster_network_tag]
+
+  # Allow all possible protocols
+  allow { protocol = "tcp" }
+  allow { protocol = "udp" }
+  allow { protocol = "icmp" }
+  allow { protocol = "sctp" }
+  allow { protocol = "esp" }
+  allow { protocol = "ah" }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_master" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master"
+  description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_endpoint_for_nodes]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["10250", "443"]
+  }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
+
+resource "google_compute_firewall" "shadow_allow_nodes" {
+  count = var.add_shadow_firewall_rules ? 1 : 0
+
+  name        = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms"
+  description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication."
+  project     = local.network_project_id
+  network     = var.network
+  priority    = var.shadow_firewall_rules_priority
+  direction   = "INGRESS"
+
+  source_ranges = [local.cluster_subnet_cidr]
+  target_tags   = [local.cluster_network_tag]
+
+  allow {
+    protocol = "icmp"
+  }
+
+  allow {
+    protocol = "udp"
+    ports    = ["1-65535"]
+  }
+
+  allow {
+    protocol = "tcp"
+    ports    = ["1-65535"]
+  }
+
+  log_config {
+    metadata = "INCLUDE_ALL_METADATA"
+  }
+}
diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf
@@ -562,6 +562,18 @@ variable "gcloud_upgrade" {
   default     = false
 }
 
+variable "add_shadow_firewall_rules" {
+  type        = bool
+  description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)."
+  default     = false
+}
+
+variable "shadow_firewall_rules_priority" {
+  type        = number
+  description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000."
+  default     = 999
+}
+
 variable "disable_default_snat" {
   type        = bool
   description = "Whether to disable the default SNAT to support the private use of public IP addresses"

diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md
@@ -133,6 +133,7 @@ Then perform the following commands on the root folder:
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no |
+| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no |
 | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no |
 | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no |
 | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no |
@@ -215,6 +216,7 @@ Then perform the following commands on the root folder:
 | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no |
 | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no |
+| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no |
 | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no |
 | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no |
 | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes |