[QT-525] enos: use spot instances for Vault targets (#20037)

The previous strategy for provisioning infrastructure targets was to use
the cheapest instances that could reliably perform as Vault cluster
nodes. With this change we introduce a new model for target node
infrastructure. We've replaced on-demand instances for a spot
fleet. While the spot price fluctuates based on dynamic pricing,
capacity, region, instance type, and platform, cost savings for our
most common combinations range between 20-70%.

This change only includes spot fleet targets for Vault clusters.
We'll be updating our Consul backend bidding in another PR.

* Create a new `vault_cluster` module that handles installation,
  configuration, initializing, and unsealing Vault clusters.
* Create a `target_ec2_instances` module that can provision a group of
  instances on-demand.
* Create a `target_ec2_spot_fleet` module that can bid on a fleet of
  spot instances.
* Extend every Enos scenario to utilize the spot fleet target acquisition
  strategy and the `vault_cluster` module.
* Update our Enos CI modules to handle both the `aws-nuke` permissions
  and also the privileges to provision spot fleets.
* Only use us-east-1 and us-west-2 in our scenario matrices as costs are
  lower than us-west-1.

Signed-off-by: Ryan Cragun <me@ryan.ec>

.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json

@@ -2,7 +2,7 @@
   "include": [
     {
       "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 3
     },
     {
@@ -12,7 +12,7 @@
     },
     {
       "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 1
     },
     {
@@ -22,7 +22,7 @@
     },
     {
       "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -32,7 +32,7 @@
     },
     {
       "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 5
     },
     {
@@ -42,7 +42,7 @@
     },
     {
       "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {

.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json

@@ -7,7 +7,7 @@
     },
     {
       "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -17,7 +17,7 @@
     },
     {
       "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 4
     },
     {
@@ -27,7 +27,7 @@
     },
     {
       "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 1
     },
     {
@@ -37,7 +37,7 @@
     },
     {
       "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 3
     },
     {
@@ -47,7 +47,7 @@
     },
     {
       "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 5
     }
   ]

.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json

@@ -2,7 +2,7 @@
   "include": [
     {
       "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -12,7 +12,7 @@
     },
     {
       "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -22,7 +22,7 @@
     },
     {
       "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -32,7 +32,7 @@
     },
     {
       "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -42,7 +42,7 @@
     },
     {
       "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {

.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json

@@ -7,17 +7,17 @@
     },
     {
       "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
       "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 1
     },
     {
       "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -27,7 +27,7 @@
     },
     {
       "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 2
     },
     {
@@ -42,7 +42,7 @@
     },
     {
       "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
-      "aws_region": "us-west-1",
+      "aws_region": "us-east-1",
       "test_group": 1
     },
     {

.gitignore

@@ -58,9 +58,17 @@ Vagrantfile
 !command/server/test-fixtures/**/*.hcl
 !enos/**/*.hcl
 
-# Enos
+# Enos local Terraform files
 enos/.enos
 enos/support
+enos/.terraform/*
+enos/.terraform.lock.hcl
+enos/*.tfstate
+enos/*.tfstate.*
+enos/**/.terraform/*
+enos/**/.terraform.lock.hcl
+enos/**/*.tfstate
+enos/**/*.tfstate.*
 
 .DS_Store
 .idea
@@ -119,3 +127,6 @@ website/components/node_modules
 
 .buildcache/
 .releaser/
+*.log
+
+tools/godoctests/.bin

enos/ci/service-user-iam/main.tf

@@ -28,6 +28,7 @@ resource "aws_iam_role" "role" {
 
 data "aws_iam_policy_document" "assume_role_policy_document" {
   provider = aws.us_east_1
+
   statement {
     effect  = "Allow"
     actions = ["sts:AssumeRole"]
@@ -43,31 +44,75 @@ resource "aws_iam_role_policy" "role_policy" {
   provider = aws.us_east_1
   role     = aws_iam_role.role.name
   name     = "${local.service_user}_policy"
-  policy   = data.aws_iam_policy_document.iam_policy_document.json
+  policy   = data.aws_iam_policy_document.role_policy.json
+}
+
+data "aws_iam_policy_document" "role_policy" {
+  source_policy_documents = [
+    data.aws_iam_policy_document.enos_scenario.json,
+    data.aws_iam_policy_document.aws_nuke.json,
+  ]
+}
+
+data "aws_iam_policy_document" "aws_nuke" {
+  provider = aws.us_east_1
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ec2:DescribeInternetGateways",
+      "ec2:DescribeNatGateways",
+      "ec2:DescribeRegions",
+      "ec2:DescribeVpnGateways",
+      "iam:DeleteAccessKey",
+      "iam:DeleteUser",
+      "iam:DeleteUserPolicy",
+      "iam:GetUser",
+      "iam:ListAccessKeys",
+      "iam:ListAccountAliases",
+      "iam:ListGroupsForUser",
+      "iam:ListUserPolicies",
+      "iam:ListUserTags",
+      "iam:ListUsers",
+      "iam:UntagUser",
+      "servicequotas:ListServiceQuotas"
+    ]
+
+    resources = ["*"]
+  }
 }
 
-data "aws_iam_policy_document" "iam_policy_document" {
+data "aws_iam_policy_document" "enos_scenario" {
   provider = aws.us_east_1
+
   statement {
     effect = "Allow"
     actions = [
       "ec2:AssociateRouteTable",
       "ec2:AttachInternetGateway",
       "ec2:AuthorizeSecurityGroupEgress",
       "ec2:AuthorizeSecurityGroupIngress",
+      "ec2:CancelSpotFleetRequests",
+      "ec2:CancelSpotInstanceRequests",
       "ec2:CreateInternetGateway",
       "ec2:CreateKeyPair",
+      "ec2:CreateLaunchTemplate",
+      "ec2:CreateLaunchTemplateVersion",
       "ec2:CreateRoute",
       "ec2:CreateRouteTable",
       "ec2:CreateSecurityGroup",
+      "ec2:CreateSpotDatafeedSubscription",
       "ec2:CreateSubnet",
       "ec2:CreateTags",
       "ec2:CreateVolume",
       "ec2:CreateVPC",
       "ec2:DeleteInternetGateway",
+      "ec2:DeleteLaunchTemplate",
+      "ec2:DeleteLaunchTemplateVersions",
       "ec2:DeleteKeyPair",
       "ec2:DeleteRouteTable",
       "ec2:DeleteSecurityGroup",
+      "ec2:DeleteSpotDatafeedSubscription",
       "ec2:DeleteSubnet",
       "ec2:DeleteTags",
       "ec2:DeleteVolume",
@@ -81,14 +126,22 @@ data "aws_iam_policy_document" "iam_policy_document" {
       "ec2:DescribeInstanceTypeOfferings",
       "ec2:DescribeInstanceTypes",
       "ec2:DescribeInternetGateways",
-      "ec2:DescribeInternetGateways",
       "ec2:DescribeKeyPairs",
+      "ec2:DescribeLaunchTemplates",
+      "ec2:DescribeLaunchTemplateVersions",
       "ec2:DescribeNatGateways",
       "ec2:DescribeNetworkAcls",
       "ec2:DescribeNetworkInterfaces",
       "ec2:DescribeRegions",
       "ec2:DescribeRouteTables",
       "ec2:DescribeSecurityGroups",
+      "ec2:DescribeSpotDatafeedSubscription",
+      "ec2:DescribeSpotFleetInstances",
+      "ec2:DescribeSpotFleetInstanceRequests",
+      "ec2:DescribeSpotFleetRequests",
+      "ec2:DescribeSpotFleetRequestHistory",
+      "ec2:DescribeSpotInstanceRequests",
+      "ec2:DescribeSpotPriceHistory",
       "ec2:DescribeSubnets",
       "ec2:DescribeTags",
       "ec2:DescribeVolumes",
@@ -99,14 +152,21 @@ data "aws_iam_policy_document" "iam_policy_document" {
       "ec2:DescribeVpnGateways",
       "ec2:DetachInternetGateway",
       "ec2:DisassociateRouteTable",
+      "ec2:GetLaunchTemplateData",
+      "ec2:GetSpotPlacementScores",
       "ec2:ImportKeyPair",
       "ec2:ModifyInstanceAttribute",
+      "ec2:ModifyLaunchTemplate",
+      "ec2:ModifySpotFleetRequest",
       "ec2:ModifySubnetAttribute",
       "ec2:ModifyVPCAttribute",
+      "ec2:RequestSpotInstances",
+      "ec2:RequestSpotFleet",
       "ec2:ResetInstanceAttribute",
       "ec2:RevokeSecurityGroupEgress",
       "ec2:RevokeSecurityGroupIngress",
       "ec2:RunInstances",
+      "ec2:SendSpotInstanceInterruptions",
       "ec2:TerminateInstances",
       "elasticloadbalancing:DescribeLoadBalancers",
       "elasticloadbalancing:DescribeTargetGroups",
@@ -115,11 +175,10 @@ data "aws_iam_policy_document" "iam_policy_document" {
       "iam:CreateInstanceProfile",
       "iam:CreatePolicy",
       "iam:CreateRole",
-      "iam:CreateRole",
+      "iam:CreateServiceLinkedRole",
       "iam:DeleteInstanceProfile",
       "iam:DeletePolicy",
       "iam:DeleteRole",
-      "iam:DeleteRole",
       "iam:DeleteRolePolicy",
       "iam:DetachRolePolicy",
       "iam:GetInstanceProfile",
@@ -132,7 +191,6 @@ data "aws_iam_policy_document" "iam_policy_document" {
       "iam:ListPolicies",
       "iam:ListRolePolicies",
       "iam:ListRoles",
-      "iam:ListRoles",
       "iam:PassRole",
       "iam:PutRolePolicy",
       "iam:RemoveRoleFromInstanceProfile",
@@ -150,6 +208,7 @@ data "aws_iam_policy_document" "iam_policy_document" {
       "kms:ScheduleKeyDeletion",
       "servicequotas:ListServiceQuotas"
     ]
+
     resources = ["*"]
   }
 }