E2E: add multi-home networking to test infrastructure

e2e/terraform/.gitignore

@@ -1 +1,2 @@
 *.zip
+uploads/

e2e/terraform/compute.tf

@@ -6,7 +6,7 @@ resource "aws_instance" "server" {
   ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.servers.id] # see also the secondary ENI
   count                  = var.server_count
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
@@ -23,7 +23,7 @@ resource "aws_instance" "client_ubuntu_jammy_amd64" {
   ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.clients.id] # see also the secondary ENI
   count                  = var.client_count_ubuntu_jammy_amd64
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
@@ -40,7 +40,7 @@ resource "aws_instance" "client_windows_2016_amd64" {
   ami                    = data.aws_ami.windows_2016_amd64.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
-  vpc_security_group_ids = [aws_security_group.primary.id]
+  vpc_security_group_ids = [aws_security_group.clients.id]
   count                  = var.client_count_windows_2016_amd64
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone

e2e/terraform/ecs.tf

@@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "nomad_rtd_e2e" {
 
 data "template_file" "ecs_vars_hcl" {
   template = <<EOT
-security_groups = ["${aws_security_group.primary.id}"]
+security_groups = ["${aws_security_group.clients.id}"]
 subnets         = ["${data.aws_subnet.default.id}"]
 EOT
 }

e2e/terraform/network.tf

@@ -5,6 +5,16 @@ data "aws_vpc" "default" {
 data "aws_subnet" "default" {
   availability_zone = var.availability_zone
   vpc_id            = data.aws_vpc.default.id
+  default_for_az    = true
+}
+
+data "aws_subnet" "secondary" {
+  availability_zone = var.availability_zone
+  vpc_id            = data.aws_vpc.default.id
+  default_for_az    = false
+  tags = {
+    Secondary = "true"
+  }
 }
 
 data "http" "my_public_ipv4" {
@@ -15,64 +25,141 @@ locals {
   ingress_cidr = var.restrict_ingress_cidrblock ? "${chomp(data.http.my_public_ipv4.body)}/32" : "0.0.0.0/0"
 }
 
-resource "aws_security_group" "primary" {
-  name   = local.random_name
+resource "aws_security_group" "servers" {
+  name   = "${local.random_name}-servers"
   vpc_id = data.aws_vpc.default.id
 
+  # SSH from test runner
   ingress {
     from_port   = 22
     to_port     = 22
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Nomad
+  # Nomad HTTP and RPC from test runner
   ingress {
     from_port   = 4646
-    to_port     = 4646
+    to_port     = 4647
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # UI reverse proxy
+  # Nomad HTTP and RPC from clients
   ingress {
-    from_port   = 6464
-    to_port     = 6464
+    from_port       = 4646
+    to_port         = 4647
+    protocol        = "tcp"
+    security_groups = [aws_security_group.clients.id]
+  }
+
+  # Nomad serf is covered here: only allowed between hosts in the servers own
+  # security group so that clients can't accidentally use serf address
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# the secondary VPC security group is intended only for internal traffic
+# and so that we can exercise behaviors with multiple IPs
+resource "aws_security_group" "servers_secondary" {
+  name   = "${local.random_name}-servers-secondary"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "clients" {
+  name   = "${local.random_name}-clients"
+  vpc_id = data.aws_vpc.default.id
+
+  # SSH from test runner
+  ingress {
+    from_port   = 22
+    to_port     = 22
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Fabio
+  # Nomad HTTP and RPC from test runner
   ingress {
-    from_port   = 9998
-    to_port     = 9999
+    from_port   = 4646
+    to_port     = 4647
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Consul: 8500 for HTTP, 8501 for HTTPS
+  # UI reverse proxy from test runner
   ingress {
-    from_port   = 8500
-    to_port     = 8501
+    from_port   = 6464
+    to_port     = 6464
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
-  # Vault
+  # Fabio from test runner
   ingress {
-    from_port   = 8200
-    to_port     = 8200
+    from_port   = 9998
+    to_port     = 9999
     protocol    = "tcp"
     cidr_blocks = [local.ingress_cidr]
   }
 
+  # allow all client-to-client
   ingress {
     from_port = 0
     to_port   = 0
     protocol  = "-1"
     self      = true
   }
 
+  # allow all outbound
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# the secondary VPC security group is intended only for internal traffic
+# and so that we can exercise behaviors with multiple IPs
+resource "aws_security_group" "clients_secondary" {
+  name   = "${local.random_name}-clients-secondary"
+  vpc_id = data.aws_vpc.default.id
+
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  # allow all outbound
   egress {
     from_port   = 0
     to_port     = 0
@@ -90,6 +177,30 @@ resource "aws_security_group" "nfs" {
     from_port       = 2049
     to_port         = 2049
     protocol        = "tcp"
-    security_groups = [aws_security_group.primary.id]
+    security_groups = [aws_security_group.clients.id]
+  }
+}
+
+# every server gets a ENI
+resource "aws_network_interface" "servers_secondary" {
+  subnet_id       = data.aws_subnet.secondary.id
+  security_groups = [aws_security_group.servers_secondary.id]
+
+  count = var.server_count
+  attachment {
+    instance     = aws_instance.server[count.index].id
+    device_index = 1
+  }
+}
+
+# every Linux client gets a ENI
+resource "aws_network_interface" "clients_secondary" {
+  subnet_id       = data.aws_subnet.secondary.id
+  security_groups = [aws_security_group.clients_secondary.id]
+
+  count = var.client_count_ubuntu_jammy_amd64
+  attachment {
+    instance     = aws_instance.client_ubuntu_jammy_amd64[count.index].id
+    device_index = 1
   }
 }