diff --git a/README.md b/README.md index 023a464a2..16a0a089d 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) |
object({
enabled = bool
min_cpu_cores = number
max_cpu_cores = number
min_memory_gb = number
max_memory_gb = number
})
|
{
"enabled": false,
"max_cpu_cores": 0,
"max_memory_gb": 0,
"min_cpu_cores": 0,
"min_memory_gb": 0
}
| no | @@ -181,6 +182,7 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl index 79069daa8..6a227cde7 100644 --- a/autogen/main/firewall.tf.tmpl +++ b/autogen/main/firewall.tf.tmpl @@ -95,3 +95,89 @@ resource "google_compute_firewall" "master_webhooks" { {% endif %} } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 8b8d19650..1505c9e58 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -580,6 +580,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + {% if beta_cluster %} variable "disable_default_snat" { type = bool diff --git a/firewall.tf b/firewall.tf index 0d0fa0718..d787d2dea 100644 --- a/firewall.tf +++ b/firewall.tf @@ -82,3 +82,89 @@ resource "google_compute_firewall" "master_webhooks" { ] } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index d0ff8ff17..5ea8ab5ed 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -155,6 +155,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | @@ -237,6 +238,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf index 4a3e64d6b..d32a964e4 100644 --- a/modules/beta-private-cluster-update-variant/firewall.tf +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -77,3 +77,89 @@ resource "google_compute_firewall" "master_webhooks" { } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 4436119ae..b338f1002 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -562,6 +562,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "disable_default_snat" { type = bool description = "Whether to disable the default SNAT to support the private use of public IP addresses" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index af93e0cde..1314d506d 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -133,6 +133,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | @@ -215,6 +216,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf index 4a3e64d6b..d32a964e4 100644 --- a/modules/beta-private-cluster/firewall.tf +++ b/modules/beta-private-cluster/firewall.tf @@ -77,3 +77,89 @@ resource "google_compute_firewall" "master_webhooks" { } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 4436119ae..b338f1002 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -562,6 +562,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "disable_default_snat" { type = bool description = "Whether to disable the default SNAT to support the private use of public IP addresses" diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 0411453cc..491be6604 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -149,6 +149,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | @@ -226,6 +227,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/beta-public-cluster-update-variant/firewall.tf b/modules/beta-public-cluster-update-variant/firewall.tf index f53ebf4e1..2bec4469c 100644 --- a/modules/beta-public-cluster-update-variant/firewall.tf +++ b/modules/beta-public-cluster-update-variant/firewall.tf @@ -83,3 +83,89 @@ resource "google_compute_firewall" "master_webhooks" { ] } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index ad3a9643b..7b69bd1fd 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -531,6 +531,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "disable_default_snat" { type = bool description = "Whether to disable the default SNAT to support the private use of public IP addresses" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index ceb1ff74d..b86a908fe 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -127,6 +127,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | @@ -204,6 +205,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf index f53ebf4e1..2bec4469c 100644 --- a/modules/beta-public-cluster/firewall.tf +++ b/modules/beta-public-cluster/firewall.tf @@ -83,3 +83,89 @@ resource "google_compute_firewall" "master_webhooks" { ] } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index ad3a9643b..7b69bd1fd 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -531,6 +531,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "disable_default_snat" { type = bool description = "Whether to disable the default SNAT to support the private use of public IP addresses" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index f43ef038e..9eeb4867e 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -152,6 +152,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) |
object({
enabled = bool
min_cpu_cores = number
max_cpu_cores = number
min_memory_gb = number
max_memory_gb = number
})
|
{
"enabled": false,
"max_cpu_cores": 0,
"max_memory_gb": 0,
"min_cpu_cores": 0,
"min_memory_gb": 0
}
| no | @@ -213,6 +214,7 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf index a3eef6b58..40ec45292 100644 --- a/modules/private-cluster-update-variant/firewall.tf +++ b/modules/private-cluster-update-variant/firewall.tf @@ -76,3 +76,89 @@ resource "google_compute_firewall" "master_webhooks" { } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 15e4d81d3..1645e5df6 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -449,6 +449,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "impersonate_service_account" { type = string diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 0277fa9d7..ed70afa9e 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -130,6 +130,7 @@ Then perform the following commands on the root folder: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | basic\_auth\_password | The password to be used with Basic Authentication. | `string` | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | `string` | `""` | no | | cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) |
object({
enabled = bool
min_cpu_cores = number
max_cpu_cores = number
min_memory_gb = number
max_memory_gb = number
})
|
{
"enabled": false,
"max_cpu_cores": 0,
"max_memory_gb": 0,
"min_cpu_cores": 0,
"min_memory_gb": 0
}
| no | @@ -191,6 +192,7 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf index a3eef6b58..40ec45292 100644 --- a/modules/private-cluster/firewall.tf +++ b/modules/private-cluster/firewall.tf @@ -76,3 +76,89 @@ resource "google_compute_firewall" "master_webhooks" { } + + +/****************************************** + Create shadow firewall rules to capture the + traffic flow between the managed firewall rules + *****************************************/ +resource "google_compute_firewall" "shadow_allow_pods" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-all" + description = "Managed by terraform gke module: A shadow firewall rule to match the default rule allowing pod communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + target_tags = [local.cluster_network_tag] + + # Allow all possible protocols + allow { protocol = "tcp" } + allow { protocol = "udp" } + allow { protocol = "icmp" } + allow { protocol = "sctp" } + allow { protocol = "esp" } + allow { protocol = "ah" } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_master" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-master" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_endpoint_for_nodes] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10250", "443"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} + +resource "google_compute_firewall" "shadow_allow_nodes" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-vms" + description = "Managed by Terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes communication." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority + direction = "INGRESS" + + source_ranges = [local.cluster_subnet_cidr] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "icmp" + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + log_config { + metadata = "INCLUDE_ALL_METADATA" + } +} diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 15e4d81d3..1645e5df6 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -449,6 +449,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "impersonate_service_account" { type = string diff --git a/variables.tf b/variables.tf index 05b1549ab..7f7dba4e6 100644 --- a/variables.tf +++ b/variables.tf @@ -425,6 +425,18 @@ variable "gcloud_upgrade" { default = false } +variable "add_shadow_firewall_rules" { + type = bool + description = "Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled)." + default = false +} + +variable "shadow_firewall_rules_priority" { + type = number + description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." + default = 999 +} + variable "impersonate_service_account" { type = string