E2E: add multi-home networking to test infrastructure

Add an Elastic Network Interface (ENI) to each Linux host, on a secondary subnet
we have provisioned in each AZ. Revise security groups as follows:

* Split out client security groups from servers so that we can't have clients
  accidentally accessing serf addresses or other unexpected cross-talk.
* Add new security groups for the secondary subnet that only allows
  communication within the security group so we can exercise behaviors with
  multiple IPs.

This changeset doesn't include any Nomad configuration changes needed to take
advantage of the extra network interface. I'll include those with testing for
PR #16217.

e2e/terraform/.gitignore

@@ -1 +1,2 @@
 *.zip
+uploads/

e2e/terraform/compute.tf

@@ -6,7 +6,7 @@ resource "aws_instance" "server" {
   ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.servers.id] # see also the secondary ENI
   count                  = var.server_count
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
@@ -23,7 +23,7 @@ resource "aws_instance" "client_ubuntu_jammy_amd64" {
   ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.clients.id] # see also the secondary ENI
   count                  = var.client_count_ubuntu_jammy_amd64
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
@@ -40,7 +40,7 @@ resource "aws_instance" "client_windows_2016_amd64" {
   ami                    = data.aws_ami.windows_2016_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.clients.id]
   count                  = var.client_count_windows_2016_amd64
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone

e2e/terraform/ecs.tf

@@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "nomad_rtd_e2e" {
 
 data "template_file" "ecs_vars_hcl" {
   template = <<EOT
-security_groups = ["${aws_security_group.primary.id}"]
+security_groups = ["${aws_security_group.clients.id}"]
 subnets         = ["${data.aws_subnet.default.id}"]
 EOT
 }

e2e/terraform/network.tf

@@ -5,6 +5,16 @@ data "aws_vpc" "default" {
 data "aws_subnet" "default" {
   availability_zone = var.availability_zone
   vpc_id            = data.aws_vpc.default.id
+  default_for_az    = true
+}
+
+data "aws_subnet" "secondary" {
+  availability_zone = var.availability_zone
+  vpc_id            = data.aws_vpc.default.id
+  default_for_az    = false
+  tags = {
+    Secondary = "true"
+  }
 }
 
 data "http" "my_public_ipv4" {
@@ -15,64 +25,141 @@ locals {
   ingress_cidr = var.restrict_ingress_cidrblock ? "${chomp(data.http.my_public_ipv4.body)}/32" : "0.0.0.0/0"
 }
 
-resource "aws_security_group" "primary" {
-  name   = local.random_name
+resource "aws_security_group" "servers" {
+  name   = "${local.random_name}-servers"
   vpc_id = data.aws_vpc.default.id
 
+  # SSH from test runner
   ingress {
     from_port   = 22
     to_port     = 22
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Nomad
+  # Nomad HTTP and RPC from test runner
   ingress {
     from_port   = 4646
-    to_port     = 4646
+    to_port     = 4647
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # UI reverse proxy
+  # Nomad HTTP and RPC from clients
   ingress {
-    from_port   = 6464
-    to_port     = 6464
+    from_port       = 4646
+    to_port         = 4647
+    protocol        = "tcp"
+    security_groups = [aws_security_group.clients.id]
+  }
+
+  # Nomad serf is covered here: only allowed between hosts in the servers own
+  # security group so that clients can't accidentally use serf address
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# the secondary VPC security group is intended only for internal traffic
+# and so that we can exercise behaviors with multiple IPs
+resource "aws_security_group" "servers_secondary" {
+  name   = "${local.random_name}-servers-secondary"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "clients" {
+  name   = "${local.random_name}-clients"
+  vpc_id = data.aws_vpc.default.id
+
+  # SSH from test runner
+  ingress {
+    from_port   = 22
+    to_port     = 22
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Fabio
+  # Nomad HTTP and RPC from test runner
   ingress {
-    from_port   = 9998
-    to_port     = 9999
+    from_port   = 4646
+    to_port     = 4647
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Consul: 8500 for HTTP, 8501 for HTTPS
+  # UI reverse proxy from test runner
   ingress {
-    from_port   = 8500
-    to_port     = 8501
+    from_port   = 6464
+    to_port     = 6464
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Vault
+  # Fabio from test runner
   ingress {
-    from_port   = 8200
-    to_port     = 8200
+    from_port   = 9998
+    to_port     = 9999
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
+  # allow all client-to-client
   ingress {
     from_port = 0
     to_port   = 0
     protocol  = "-1"
     self      = true
   }
 
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# the secondary VPC security group is intended only for internal traffic
+# and so that we can exercise behaviors with multiple IPs
+resource "aws_security_group" "clients_secondary" {
+  name   = "${local.random_name}-clients-secondary"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
   egress {
     from_port   = 0
     to_port     = 0
@@ -90,6 +177,30 @@ resource "aws_security_group" "nfs" {
     from_port       = 2049
     to_port         = 2049
     protocol        = "tcp"
-    security_groups = [aws_security_group.primary.id]
+    security_groups = [aws_security_group.clients.id]
+  }
+}
+
+# every server gets a ENI
+resource "aws_network_interface" "servers_secondary" {
+  subnet_id       = data.aws_subnet.secondary.id
+  security_groups = [aws_security_group.servers_secondary.id]
+
+  count = var.server_count
+  attachment {
+    instance     = aws_instance.server[count.index].id
+    device_index = 1
+  }
+}
+
+# every Linux client gets a ENI
+resource "aws_network_interface" "clients_secondary" {
+  subnet_id       = data.aws_subnet.secondary.id
+  security_groups = [aws_security_group.clients_secondary.id]
+
+  count = var.client_count_ubuntu_jammy_amd64
+  attachment {
+    instance     = aws_instance.client_ubuntu_jammy_amd64[count.index].id
+    device_index = 1
   }
 }