From 2a0c73a91be3b19ae30b9a07fefa518079e29535 Mon Sep 17 00:00:00 2001
From: Tim Gross <tgross@hashicorp.com>
Date: Fri, 17 Feb 2023 16:25:33 -0500
Subject: [PATCH] E2E: add multi-home networking to test infrastructure

Add an Elastic Network Interface (ENI) to each Linux host, on a secondary subnet
we have provisioned in each AZ. Revise security groups as follows:

* Split out client security groups from servers so that we can't have clients
  accidentally accessing serf addresses or other unexpected cross-talk.
* Add new security groups for the secondary subnet that only allows
  communication within the security group so we can exercise behaviors with
  multiple IPs.

This changeset doesn't include any Nomad configuration changes needed to take
advantage of the extra network interface. I'll include those with testing for
PR #16217.
---
 e2e/terraform/.gitignore          |   1 +
 e2e/terraform/.terraform.lock.hcl |  11 +++
 e2e/terraform/compute.tf          |   6 +-
 e2e/terraform/ecs.tf              |   2 +-
 e2e/terraform/network.tf          | 145 ++++++++++++++++++++++++++----
 5 files changed, 144 insertions(+), 21 deletions(-)

diff --git a/e2e/terraform/.gitignore b/e2e/terraform/.gitignore
index c4c4ffc6aa41..cfbd9940db2a 100644
--- a/e2e/terraform/.gitignore
+++ b/e2e/terraform/.gitignore
@@ -1 +1,2 @@
 *.zip
+uploads/
diff --git a/e2e/terraform/.terraform.lock.hcl b/e2e/terraform/.terraform.lock.hcl
index 941992745068..f2fd0472ac9e 100644
--- a/e2e/terraform/.terraform.lock.hcl
+++ b/e2e/terraform/.terraform.lock.hcl
@@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/aws" {
   version = "4.10.0"
   hashes = [
     "h1:F9BjbxBhuo1A/rP318IUrkW3TAh29i6UC18qwhzCs6c=",
+    "h1:S6xGPRL08YEuBdemiYZyIBf/YwM4OCvzVuaiuU6kLjc=",
     "zh:0a2a7eabfeb7dbb17b7f82aff3fa2ba51e836c15e5be4f5468ea44bd1299b48d",
     "zh:23409c7205d13d2d68b5528e1c49e0a0455d99bbfec61eb0201142beffaa81f7",
     "zh:3adad2245d97816f3919778b52c58fb2de130938a3e9081358bfbb72ec478d9a",
@@ -23,6 +24,7 @@ provider "registry.terraform.io/hashicorp/aws" {
 provider "registry.terraform.io/hashicorp/consul" {
   version = "2.15.1"
   hashes = [
+    "h1:PexyQBRLDA+SR+sWlzYBZswry5O5h/tTfj87CaECtLc=",
     "h1:XGOBrMc6OQsNpgQtgtV6H0/jYe7yVIYxEDsErV/R6SE=",
     "zh:1806830a3cf103e65e772a7d28fd4df2788c29a029fb2def1326bc777ad107ed",
     "zh:252be544fb4c9daf09cad7d3776daf5fa66b62740d3ea9d6d499a7b1697c3433",
@@ -43,6 +45,7 @@ provider "registry.terraform.io/hashicorp/external" {
   version = "2.2.2"
   hashes = [
     "h1:VUkgcWvCliS0HO4kt7oEQhFD2gcx/59XpwMqxfCU1kE=",
+    "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=",
     "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca",
     "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28",
     "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b",
@@ -61,6 +64,7 @@ provider "registry.terraform.io/hashicorp/external" {
 provider "registry.terraform.io/hashicorp/hcp" {
   version = "0.26.0"
   hashes = [
+    "h1:B5O/NawTnKPdUgUlGP/mM2ybv0RcLvVJVOcrivDdFnI=",
     "h1:C0KoYT09Ff91pE5KzrFrISCE5wQyJaJnxPdA0SXDOzI=",
     "zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d",
     "zh:6fa5415dbac9c8d20026772dd5aee7dd3ac541e9d86827d0b70bc752472ec76c",
@@ -81,6 +85,7 @@ provider "registry.terraform.io/hashicorp/http" {
   version = "2.1.0"
   hashes = [
     "h1:GYoVrTtiSAE3AlP1fad3fFmHoPaXAPhm/DJyMcVCwZA=",
+    "h1:HmUcHqc59VeHReHD2SEhnLVQPUKHKTipJ8Jxq67GiDU=",
     "h1:q/YFxlfQW6FAMM5LIITGWnlJIuu52eqij82TLp135x8=",
     "zh:03d82dc0887d755b8406697b1d27506bc9f86f93b3e9b4d26e0679d96b802826",
     "zh:0704d02926393ddc0cfad0b87c3d51eafeeae5f9e27cc71e193c141079244a22",
@@ -99,6 +104,7 @@ provider "registry.terraform.io/hashicorp/http" {
 provider "registry.terraform.io/hashicorp/local" {
   version = "2.2.2"
   hashes = [
+    "h1:5UYW2wJ320IggrzLt8tLD6MowePqycWtH1b2RInHZkE=",
     "h1:SjDyZXIUHEQzZe10VjhlhZq2a9kgQB6tmqJcpq2BeWg=",
     "zh:027e4873c69da214e2fed131666d5de92089732a11d096b68257da54d30b6f9d",
     "zh:0ba2216e16cfb72538d76a4c4945b4567a76f7edbfef926b1c5a08d7bba2a043",
@@ -118,6 +124,7 @@ provider "registry.terraform.io/hashicorp/local" {
 provider "registry.terraform.io/hashicorp/null" {
   version = "3.1.1"
   hashes = [
+    "h1:71sNUDvmiJcijsvfXpiLCz0lXIBSsEJjMxljt7hxMhw=",
     "h1:Pctug/s/2Hg5FJqjYcTM0kPyx3AoYK1MpRWO0T9V2ns=",
     "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597",
     "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf",
@@ -137,6 +144,7 @@ provider "registry.terraform.io/hashicorp/null" {
 provider "registry.terraform.io/hashicorp/random" {
   version = "3.1.2"
   hashes = [
+    "h1:5A5VsY5wNmOZlupUcLnIoziMPn8htSZBXbP3lI7lBEM=",
     "h1:9A6Ghjgad0KjJRxa6nPo8i8uFvwj3Vv0wnEgy49u+24=",
     "zh:0daceba867b330d3f8e2c5dc895c4291845a78f31955ce1b91ab2c4d1cd1c10b",
     "zh:104050099efd30a630741f788f9576b19998e7a09347decbec3da0b21d64ba2d",
@@ -158,6 +166,7 @@ provider "registry.terraform.io/hashicorp/template" {
   hashes = [
     "h1:0PGmlQJDT2HHYSryvhnhvd9P5UzMZ3KX3YyMNsOYXU0=",
     "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=",
+    "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=",
     "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386",
     "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53",
     "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603",
@@ -175,6 +184,7 @@ provider "registry.terraform.io/hashicorp/tls" {
   version = "3.3.0"
   hashes = [
     "h1:A4xOtHhD4jCmn4nO1xCTk2Nl5IP5JpjicjF+Fuu2ZFQ=",
+    "h1:xx/b39Q9FVZSlDc97rlDmQ9dNaaxFFyVzP9kV+47z28=",
     "zh:16140e8cc880f95b642b6bf6564f4e98760e9991864aacc8e21273423571e561",
     "zh:16338b8457759c97fdd73153965d6063b037f2954fd512e569fcdc42b7fef743",
     "zh:348bd44b7cd0c6d663bba36cecb474c17635a8f22b02187d034b8e57a8729c5a",
@@ -194,6 +204,7 @@ provider "registry.terraform.io/hashicorp/vault" {
   version = "3.4.1"
   hashes = [
     "h1:HIjd/7KktGO5E/a0uICbIanUj0Jdd0j8aL/r+QxFhAs=",
+    "h1:dXJBo807u69+Uib2hjoBQ68G2+nGXcNZeq/THVyQQVc=",
     "zh:1eb8370a1846e34e2bcc4d11eece5733735784a8eab447bbed3cfd822101b577",
     "zh:2df3989327cea68b2167514b7ebddc67b09340f00bbf3fa85df03c97adfb9d25",
     "zh:3dd1e317264f574985e856296deef71a76464918bf0566eb0d7f6389ea0586bd",
diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf
index c8b1908b4362..835c3eeaa92f 100644
--- a/e2e/terraform/compute.tf
+++ b/e2e/terraform/compute.tf
@@ -6,7 +6,7 @@ resource "aws_instance" "server" {
   ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.servers.id] # see also the secondary ENI
   count                  = var.server_count
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
@@ -23,7 +23,7 @@ resource "aws_instance" "client_ubuntu_jammy_amd64" {
   ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.clients.id] # see also the secondary ENI
   count                  = var.client_count_ubuntu_jammy_amd64
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
@@ -40,7 +40,7 @@ resource "aws_instance" "client_windows_2016_amd64" {
   ami                    = data.aws_ami.windows_2016_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.clients.id]
   count                  = var.client_count_windows_2016_amd64
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
diff --git a/e2e/terraform/ecs.tf b/e2e/terraform/ecs.tf
index 121424a7498f..20d764c89822 100644
--- a/e2e/terraform/ecs.tf
+++ b/e2e/terraform/ecs.tf
@@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "nomad_rtd_e2e" {
 
 data "template_file" "ecs_vars_hcl" {
   template = <<EOT
-security_groups = ["${aws_security_group.primary.id}"]
+security_groups = ["${aws_security_group.clients.id}"]
 subnets         = ["${data.aws_subnet.default.id}"]
 EOT
 }
diff --git a/e2e/terraform/network.tf b/e2e/terraform/network.tf
index 79cd6b73c2ae..26b62023b66a 100644
--- a/e2e/terraform/network.tf
+++ b/e2e/terraform/network.tf
@@ -5,6 +5,16 @@ data "aws_vpc" "default" {
 data "aws_subnet" "default" {
   availability_zone = var.availability_zone
   vpc_id            = data.aws_vpc.default.id
+  default_for_az    = true
+}
+
+data "aws_subnet" "secondary" {
+  availability_zone = var.availability_zone
+  vpc_id            = data.aws_vpc.default.id
+  default_for_az    = false
+  tags = {
+    Secondary = "true"
+  }
 }
 
 data "http" "my_public_ipv4" {
@@ -15,10 +25,11 @@ locals {
   ingress_cidr = var.restrict_ingress_cidrblock ? "${chomp(data.http.my_public_ipv4.body)}/32" : "0.0.0.0/0"
 }
 
-resource "aws_security_group" "primary" {
-  name   = local.random_name
+resource "aws_security_group" "servers" {
+  name   = "${local.random_name}-servers"
   vpc_id = data.aws_vpc.default.id
 
+  # SSH from test runner
   ingress {
     from_port   = 22
     to_port     = 22
@@ -26,46 +37,99 @@ resource "aws_security_group" "primary" {
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Nomad
+  # Nomad HTTP and RPC from test runner
   ingress {
     from_port   = 4646
-    to_port     = 4646
+    to_port     = 4647
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # UI reverse proxy
+  # Nomad HTTP and RPC from clients
   ingress {
-    from_port   = 6464
-    to_port     = 6464
+    from_port       = 4646
+    to_port         = 4647
+    protocol        = "tcp"
+    security_groups = [aws_security_group.clients.id]
+  }
+
+  # Nomad serf is covered here: only allowed between hosts in the servers own
+  # security group so that clients can't accidentally use serf address
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# the secondary VPC security group is intended only for internal traffic
+# and so that we can exercise behaviors with multiple IPs
+resource "aws_security_group" "servers_secondary" {
+  name   = "${local.random_name}-servers-secondary"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "clients" {
+  name   = "${local.random_name}-clients"
+  vpc_id = data.aws_vpc.default.id
+
+  # SSH from test runner
+  ingress {
+    from_port   = 22
+    to_port     = 22
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Fabio
+  # Nomad HTTP and RPC from test runner
   ingress {
-    from_port   = 9998
-    to_port     = 9999
+    from_port   = 4646
+    to_port     = 4647
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Consul: 8500 for HTTP, 8501 for HTTPS
+  # UI reverse proxy from test runner
   ingress {
-    from_port   = 8500
-    to_port     = 8501
+    from_port   = 6464
+    to_port     = 6464
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Vault
+  # Fabio from test runner
   ingress {
-    from_port   = 8200
-    to_port     = 8200
+    from_port   = 9998
+    to_port     = 9999
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
+  # allow all client-to-client
   ingress {
     from_port = 0
     to_port   = 0
@@ -73,6 +137,29 @@ resource "aws_security_group" "primary" {
     self      = true
   }
 
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# the secondary VPC security group is intended only for internal traffic
+# and so that we can exercise behaviors with multiple IPs
+resource "aws_security_group" "clients_secondary" {
+  name   = "${local.random_name}-clients-secondary"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
   egress {
     from_port   = 0
     to_port     = 0
@@ -90,6 +177,30 @@ resource "aws_security_group" "nfs" {
     from_port       = 2049
     to_port         = 2049
     protocol        = "tcp"
-    security_groups = [aws_security_group.primary.id]
+    security_groups = [aws_security_group.clients.id]
+  }
+}
+
+# every server gets a ENI
+resource "aws_network_interface" "servers_secondary" {
+  subnet_id       = data.aws_subnet.secondary.id
+  security_groups = [aws_security_group.servers_secondary.id]
+
+  count = var.server_count
+  attachment {
+    instance     = aws_instance.server[count.index].id
+    device_index = 1
+  }
+}
+
+# every Linux client gets a ENI
+resource "aws_network_interface" "clients_secondary" {
+  subnet_id       = data.aws_subnet.secondary.id
+  security_groups = [aws_security_group.clients_secondary.id]
+
+  count = var.client_count_ubuntu_jammy_amd64
+  attachment {
+    instance     = aws_instance.client_ubuntu_jammy_amd64[count.index].id
+    device_index = 1
   }
 }