From 3f2758c5fbb98f630de11dc87c24e89f18c3d0f1 Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Sun, 23 Apr 2023 16:25:32 -0600 Subject: [PATCH] [QT-530] enos: allow-list all public IP addresses (#20304) The security groups that allow access to remote machines in Enos scenarios have been configured to only allow port 22 (SSH) from the public IP address of machine executing the Enos scenario. To achieve this we previously utilized the `enos_environment.public_ip_address` attribute. Sometime in mid March we started seeing sporadic SSH i/o timeout errors when attempting to execute Enos resources against SSH transport targets. We've only ever seen this when communicating from Azure hosted runners to AWS hosted machines. While testing we were able to confirm that in some cases the public IP address resolved using DNS over UDP4 to Google and OpenDNS name servers did not match what was resolved when using the HTTPS/TCP IP address service hosted by AWS. The Enos data source was implemented in a way that we'd attempt resolution of a single name server and only attempt resolving from the next if previous name server could not get a result. We'd then allow-list that single IP address. That's a problem if we can resolve two different public IP addresses depending our endpoint address. This change utlizes the new `enos_environment.public_ip_addresses` attribute and subsequent behavior change. Now the data source will attempt to resolve our public IP address via name servers hosted by Google, OpenDNS, Cloudflare, and AWS. We then return a unique set of these IP addresses and allow-list all of them in our security group. It is our hope that this resolves these i/o timeout errors that seem like they're caused by the security group black-holing our attempted access because the IP we resolved does not match what we're actually exiting with. Signed-off-by: Ryan Cragun --- enos/modules/target_ec2_instances/main.tf | 40 ++++++++++++++-------- enos/modules/target_ec2_spot_fleet/main.tf | 22 ++++++------ 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/enos/modules/target_ec2_instances/main.tf b/enos/modules/target_ec2_instances/main.tf index 06b44aad21f7..26a520d3e8ce 100644 --- a/enos/modules/target_ec2_instances/main.tf +++ b/enos/modules/target_ec2_instances/main.tf @@ -4,7 +4,7 @@ terraform { # to the public registry enos = { source = "app.terraform.io/hashicorp-qti/enos" - version = ">= 0.3.2" + version = ">= 0.3.24" } } } @@ -97,10 +97,13 @@ resource "aws_security_group" "target" { # SSH traffic ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)] + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) } # Vault traffic @@ -109,24 +112,31 @@ resource "aws_security_group" "target" { to_port = 8201 protocol = "tcp" cidr_blocks = flatten([ - "${data.enos_environment.localhost.public_ip_address}/32", + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), - formatlist("%s/32", var.ssh_allow_ips)]) + formatlist("%s/32", var.ssh_allow_ips) + ]) } # Consul traffic ingress { - from_port = 8301 - to_port = 8301 - protocol = "tcp" - cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)] + from_port = 8301 + to_port = 8301 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) } ingress { - from_port = 8301 - to_port = 8301 - protocol = "udp" - cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)] + from_port = 8301 + to_port = 8301 + protocol = "udp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) } # Internal traffic diff --git a/enos/modules/target_ec2_spot_fleet/main.tf b/enos/modules/target_ec2_spot_fleet/main.tf index 56c786f5c6a1..4e55da2dd095 100644 --- a/enos/modules/target_ec2_spot_fleet/main.tf +++ b/enos/modules/target_ec2_spot_fleet/main.tf @@ -4,7 +4,7 @@ terraform { # to the public registry enos = { source = "app.terraform.io/hashicorp-qti/enos" - version = ">= 0.3.2" + version = ">= 0.3.24" } } } @@ -209,10 +209,10 @@ resource "aws_security_group" "target" { from_port = 22 to_port = 22 protocol = "tcp" - cidr_blocks = [ - "${data.enos_environment.localhost.public_ip_address}/32", + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), - ] + ]) } # Vault traffic @@ -221,7 +221,7 @@ resource "aws_security_group" "target" { to_port = 8201 protocol = "tcp" cidr_blocks = flatten([ - "${data.enos_environment.localhost.public_ip_address}/32", + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), formatlist("%s/32", var.ssh_allow_ips) ]) @@ -232,20 +232,20 @@ resource "aws_security_group" "target" { from_port = 8301 to_port = 8301 protocol = "tcp" - cidr_blocks = [ - "${data.enos_environment.localhost.public_ip_address}/32", + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), - ] + ]) } ingress { from_port = 8301 to_port = 8301 protocol = "udp" - cidr_blocks = [ - "${data.enos_environment.localhost.public_ip_address}/32", + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), - ] + ]) } # Internal traffic