From cc9cdfb5c863b73a50e8f05c0f2f00d711ec0b19 Mon Sep 17 00:00:00 2001 From: hc-github-team-secure-vault-core <82990506+hc-github-team-secure-vault-core@users.noreply.github.com> Date: Sun, 23 Apr 2023 19:17:04 -0400 Subject: [PATCH] Backport of [QT-525] and [QT-530] into release/1.11.x (#20160) * [QT-525] enos: use spot instances for Vault targets (#20037) The previous strategy for provisioning infrastructure targets was to use the cheapest instances that could reliably perform as Vault cluster nodes. With this change we introduce a new model for target node infrastructure. We've replaced on-demand instances for a spot fleet. While the spot price fluctuates based on dynamic pricing, capacity, region, instance type, and platform, cost savings for our most common combinations range between 20-70%. This change only includes spot fleet targets for Vault clusters. We'll be updating our Consul backend bidding in another PR. * Create a new `vault_cluster` module that handles installation, configuration, initializing, and unsealing Vault clusters. * Create a `target_ec2_instances` module that can provision a group of instances on-demand. * Create a `target_ec2_spot_fleet` module that can bid on a fleet of spot instances. * Extend every Enos scenario to utilize the spot fleet target acquisition strategy and the `vault_cluster` module. * Update our Enos CI modules to handle both the `aws-nuke` permissions and also the privileges to provision spot fleets. * Only use us-east-1 and us-west-2 in our scenario matrices as costs are lower than us-west-1. Signed-off-by: Ryan Cragun * [QT-530] enos: allow-list all public IP addresses (#20304) The security groups that allow access to remote machines in Enos scenarios have been configured to only allow port 22 (SSH) from the public IP address of machine executing the Enos scenario. To achieve this we previously utilized the `enos_environment.public_ip_address` attribute. Sometime in mid March we started seeing sporadic SSH i/o timeout errors when attempting to execute Enos resources against SSH transport targets. We've only ever seen this when communicating from Azure hosted runners to AWS hosted machines. While testing we were able to confirm that in some cases the public IP address resolved using DNS over UDP4 to Google and OpenDNS name servers did not match what was resolved when using the HTTPS/TCP IP address service hosted by AWS. The Enos data source was implemented in a way that we'd attempt resolution of a single name server and only attempt resolving from the next if previous name server could not get a result. We'd then allow-list that single IP address. That's a problem if we can resolve two different public IP addresses depending our endpoint address. This change utlizes the new `enos_environment.public_ip_addresses` attribute and subsequent behavior change. Now the data source will attempt to resolve our public IP address via name servers hosted by Google, OpenDNS, Cloudflare, and AWS. We then return a unique set of these IP addresses and allow-list all of them in our security group. It is our hope that this resolves these i/o timeout errors that seem like they're caused by the security group black-holing our attempted access because the IP we resolved does not match what we're actually exiting with. Signed-off-by: Ryan Cragun --------- Signed-off-by: Ryan Cragun Co-authored-by: Ryan Cragun --- .../build-github-oss-linux-amd64-zip.json | 10 +- .../build-github-oss-linux-arm64-zip.json | 10 +- ...g_oss-artifactory-oss-linux-amd64-zip.json | 10 +- ...g_oss-artifactory-oss-linux-arm64-zip.json | 10 +- .gitignore | 13 +- enos/ci/service-user-iam/main.tf | 71 +++- enos/ci/service-user-iam/service-quotas.tf | 39 +- enos/enos-modules.hcl | 32 +- enos/enos-scenario-agent.hcl | 148 ++++--- enos/enos-scenario-autopilot.hcl | 248 ++++++----- enos/enos-scenario-replication.hcl | 354 +++++++++------- enos/enos-scenario-smoke.hcl | 140 ++++--- enos/enos-scenario-ui.hcl | 117 ++++-- enos/enos-scenario-upgrade.hcl | 139 ++++--- enos/modules/target_ec2_instances/main.tf | 181 ++++++++ enos/modules/target_ec2_instances/outputs.tf | 11 + .../modules/target_ec2_instances/variables.tf | 61 +++ enos/modules/target_ec2_spot_fleet/main.tf | 388 ++++++++++++++++++ enos/modules/target_ec2_spot_fleet/outputs.tf | 11 + .../target_ec2_spot_fleet/variables.tf | 88 ++++ enos/modules/vault_cluster/main.tf | 335 +++++++++++++++ enos/modules/vault_cluster/outputs.tf | 55 +++ .../templates/install-packages.sh | 44 ++ .../templates/vault-write-license.sh | 38 ++ enos/modules/vault_cluster/variables.tf | 176 ++++++++ 25 files changed, 2214 insertions(+), 515 deletions(-) create mode 100644 enos/modules/target_ec2_instances/main.tf create mode 100644 enos/modules/target_ec2_instances/outputs.tf create mode 100644 enos/modules/target_ec2_instances/variables.tf create mode 100644 enos/modules/target_ec2_spot_fleet/main.tf create mode 100644 enos/modules/target_ec2_spot_fleet/outputs.tf create mode 100644 enos/modules/target_ec2_spot_fleet/variables.tf create mode 100644 enos/modules/vault_cluster/main.tf create mode 100644 enos/modules/vault_cluster/outputs.tf create mode 100755 enos/modules/vault_cluster/templates/install-packages.sh create mode 100755 enos/modules/vault_cluster/templates/vault-write-license.sh create mode 100644 enos/modules/vault_cluster/variables.tf diff --git a/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json b/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json index ab09a413bad3..80b3d55212b2 100644 --- a/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json +++ b/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json @@ -2,7 +2,7 @@ "include": [ { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 3 }, { @@ -12,7 +12,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { @@ -22,7 +22,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -32,7 +32,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 5 }, { @@ -42,7 +42,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { diff --git a/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json b/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json index ec951fdd0a18..a497fb0ebe00 100644 --- a/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json +++ b/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json @@ -7,7 +7,7 @@ }, { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -17,7 +17,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 4 }, { @@ -27,7 +27,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { @@ -37,7 +37,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 3 }, { @@ -47,7 +47,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 5 } ] diff --git a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json index 70e5ea1c3c24..857677b72f07 100644 --- a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json +++ b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json @@ -2,7 +2,7 @@ "include": [ { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -12,7 +12,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -22,7 +22,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -32,7 +32,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -42,7 +42,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { diff --git a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json index e6e9edb10f28..1c67cd3bcfdb 100644 --- a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json +++ b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json @@ -7,17 +7,17 @@ }, { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -27,7 +27,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -42,7 +42,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { diff --git a/.gitignore b/.gitignore index e66eb7121fda..923efe7e4189 100644 --- a/.gitignore +++ b/.gitignore @@ -58,9 +58,17 @@ Vagrantfile !command/server/test-fixtures/**/*.hcl !enos/**/*.hcl -# Enos +# Enos local Terraform files enos/.enos enos/support +enos/.terraform/* +enos/.terraform.lock.hcl +enos/*.tfstate +enos/*.tfstate.* +enos/**/.terraform/* +enos/**/.terraform.lock.hcl +enos/**/*.tfstate +enos/**/*.tfstate.* .DS_Store .idea @@ -119,3 +127,6 @@ website/components/node_modules .buildcache/ .releaser/ +*.log + +tools/godoctests/.bin diff --git a/enos/ci/service-user-iam/main.tf b/enos/ci/service-user-iam/main.tf index a42333932267..bea2d46a4309 100644 --- a/enos/ci/service-user-iam/main.tf +++ b/enos/ci/service-user-iam/main.tf @@ -28,6 +28,7 @@ resource "aws_iam_role" "role" { data "aws_iam_policy_document" "assume_role_policy_document" { provider = aws.us_east_1 + statement { effect = "Allow" actions = ["sts:AssumeRole"] @@ -43,11 +44,47 @@ resource "aws_iam_role_policy" "role_policy" { provider = aws.us_east_1 role = aws_iam_role.role.name name = "${local.service_user}_policy" - policy = data.aws_iam_policy_document.iam_policy_document.json + policy = data.aws_iam_policy_document.role_policy.json +} + +data "aws_iam_policy_document" "role_policy" { + source_policy_documents = [ + data.aws_iam_policy_document.enos_scenario.json, + data.aws_iam_policy_document.aws_nuke.json, + ] +} + +data "aws_iam_policy_document" "aws_nuke" { + provider = aws.us_east_1 + + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInternetGateways", + "ec2:DescribeNatGateways", + "ec2:DescribeRegions", + "ec2:DescribeVpnGateways", + "iam:DeleteAccessKey", + "iam:DeleteUser", + "iam:DeleteUserPolicy", + "iam:GetUser", + "iam:ListAccessKeys", + "iam:ListAccountAliases", + "iam:ListGroupsForUser", + "iam:ListUserPolicies", + "iam:ListUserTags", + "iam:ListUsers", + "iam:UntagUser", + "servicequotas:ListServiceQuotas" + ] + + resources = ["*"] + } } -data "aws_iam_policy_document" "iam_policy_document" { +data "aws_iam_policy_document" "enos_scenario" { provider = aws.us_east_1 + statement { effect = "Allow" actions = [ @@ -55,19 +92,27 @@ data "aws_iam_policy_document" "iam_policy_document" { "ec2:AttachInternetGateway", "ec2:AuthorizeSecurityGroupEgress", "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotFleetRequests", + "ec2:CancelSpotInstanceRequests", "ec2:CreateInternetGateway", "ec2:CreateKeyPair", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", "ec2:CreateRoute", "ec2:CreateRouteTable", "ec2:CreateSecurityGroup", + "ec2:CreateSpotDatafeedSubscription", "ec2:CreateSubnet", "ec2:CreateTags", "ec2:CreateVolume", "ec2:CreateVPC", "ec2:DeleteInternetGateway", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", "ec2:DeleteKeyPair", "ec2:DeleteRouteTable", "ec2:DeleteSecurityGroup", + "ec2:DeleteSpotDatafeedSubscription", "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DeleteVolume", @@ -81,14 +126,22 @@ data "aws_iam_policy_document" "iam_policy_document" { "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeInstanceTypes", "ec2:DescribeInternetGateways", - "ec2:DescribeInternetGateways", "ec2:DescribeKeyPairs", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", "ec2:DescribeNatGateways", "ec2:DescribeNetworkAcls", "ec2:DescribeNetworkInterfaces", "ec2:DescribeRegions", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotDatafeedSubscription", + "ec2:DescribeSpotFleetInstances", + "ec2:DescribeSpotFleetInstanceRequests", + "ec2:DescribeSpotFleetRequests", + "ec2:DescribeSpotFleetRequestHistory", + "ec2:DescribeSpotInstanceRequests", + "ec2:DescribeSpotPriceHistory", "ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVolumes", @@ -99,14 +152,21 @@ data "aws_iam_policy_document" "iam_policy_document" { "ec2:DescribeVpnGateways", "ec2:DetachInternetGateway", "ec2:DisassociateRouteTable", + "ec2:GetLaunchTemplateData", + "ec2:GetSpotPlacementScores", "ec2:ImportKeyPair", "ec2:ModifyInstanceAttribute", + "ec2:ModifyLaunchTemplate", + "ec2:ModifySpotFleetRequest", "ec2:ModifySubnetAttribute", "ec2:ModifyVPCAttribute", + "ec2:RequestSpotInstances", + "ec2:RequestSpotFleet", "ec2:ResetInstanceAttribute", "ec2:RevokeSecurityGroupEgress", "ec2:RevokeSecurityGroupIngress", "ec2:RunInstances", + "ec2:SendSpotInstanceInterruptions", "ec2:TerminateInstances", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeTargetGroups", @@ -115,11 +175,10 @@ data "aws_iam_policy_document" "iam_policy_document" { "iam:CreateInstanceProfile", "iam:CreatePolicy", "iam:CreateRole", - "iam:CreateRole", + "iam:CreateServiceLinkedRole", "iam:DeleteInstanceProfile", "iam:DeletePolicy", "iam:DeleteRole", - "iam:DeleteRole", "iam:DeleteRolePolicy", "iam:DetachRolePolicy", "iam:GetInstanceProfile", @@ -132,7 +191,6 @@ data "aws_iam_policy_document" "iam_policy_document" { "iam:ListPolicies", "iam:ListRolePolicies", "iam:ListRoles", - "iam:ListRoles", "iam:PassRole", "iam:PutRolePolicy", "iam:RemoveRoleFromInstanceProfile", @@ -150,6 +208,7 @@ data "aws_iam_policy_document" "iam_policy_document" { "kms:ScheduleKeyDeletion", "servicequotas:ListServiceQuotas" ] + resources = ["*"] } } diff --git a/enos/ci/service-user-iam/service-quotas.tf b/enos/ci/service-user-iam/service-quotas.tf index 73a68363d84d..544f311504e7 100644 --- a/enos/ci/service-user-iam/service-quotas.tf +++ b/enos/ci/service-user-iam/service-quotas.tf @@ -1,33 +1,62 @@ locals { // This is the code of the service quota to request a change for. Each adjustable limit has a // unique code. See, https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicequotas_service_quota#quota_code - subnets_per_vps_quota = "L-F678F1CE" + subnets_per_vpcs_quota = "L-F678F1CE" + standard_spot_instance_requests_quota = "L-34B43A08" } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_1" { provider = aws.us_east_2 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_2" { provider = aws.us_east_2 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_1" { provider = aws.us_west_1 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_2" { provider = aws.us_west_2 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_1" { + provider = aws.us_east_2 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_2" { + provider = aws.us_east_2 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_1" { + provider = aws.us_west_1 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_2" { + provider = aws.us_west_2 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 26cb20dac4d6..9216fbf8f215 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -65,6 +65,27 @@ module "shutdown_multiple_nodes" { source = "./modules/shutdown_multiple_nodes" } +module "target_ec2_instances" { + source = "./modules/target_ec2_instances" + + common_tags = var.tags + instance_count = var.vault_instance_count + project_name = var.project_name + ssh_keypair = var.aws_ssh_keypair_name +} + +module "target_ec2_spot_fleet" { + source = "./modules/target_ec2_spot_fleet" + + common_tags = var.tags + instance_mem_min = 4096 + instance_cpu_min = 2 + project_name = var.project_name + // Current on-demand cost of t3.medium in us-east. + spot_price_max = "0.0416" + ssh_keypair = var.aws_ssh_keypair_name +} + module "vault_agent" { source = "./modules/vault_agent" @@ -72,7 +93,6 @@ module "vault_agent" { vault_instance_count = var.vault_instance_count } - module "vault_verify_agent_output" { source = "./modules/vault_verify_agent_output" @@ -80,15 +100,9 @@ module "vault_verify_agent_output" { } module "vault_cluster" { - source = "app.terraform.io/hashicorp-qti/aws-vault/enos" - # source = "../../terraform-enos-aws-vault" + source = "./modules/vault_cluster" - common_tags = var.tags - environment = "ci" - instance_count = var.vault_instance_count - project_name = var.project_name - ssh_aws_keypair = var.aws_ssh_keypair_name - vault_install_dir = var.vault_install_dir + install_dir = var.vault_install_dir } module "vault_get_cluster_ips" { diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 8a7de6032dac..29a1204aae83 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -22,13 +22,18 @@ scenario "agent" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } install_artifactory_artifact = local.bundle_path == null + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -51,21 +56,21 @@ scenario "agent" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) - bundle_path = local.bundle_path - goarch = matrix.arch - goos = "linux" - artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null - artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null - artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null - artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null - arch = matrix.artifact_source == "artifactory" ? matrix.arch : null - vault_product_version = var.vault_product_version - artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null - distro = matrix.artifact_source == "artifactory" ? matrix.distro : null - edition = matrix.artifact_source == "artifactory" ? matrix.edition : null - instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null - revision = var.vault_revision + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + bundle_path = local.bundle_path + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null + artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null + artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.artifact_source == "artifactory" ? matrix.arch : null + product_version = var.vault_product_version + artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null + distro = matrix.artifact_source == "artifactory" ? matrix.distro : null + edition = matrix.artifact_source == "artifactory" ? matrix.edition : null + instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null + revision = var.vault_revision } } @@ -99,28 +104,29 @@ scenario "agent" { } } - step "create_backend_cluster" { - module = "backend_raft" + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances depends_on = [step.create_vpc] providers = { - enos = provider.enos.ubuntu + enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids["ubuntu"]["amd64"] - common_tags = local.tags - instance_type = var.backend_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - vpc_id = step.create_vpc.vpc_id + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id } } step "create_vault_cluster" { module = module.vault_cluster depends_on = [ - step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -128,28 +134,25 @@ scenario "agent" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags - consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = "raft" - unseal_method = "shamir" - vault_local_artifact_path = local.bundle_path - vault_artifactory_release = local.install_artifactory_artifact ? step.build_vault.vault_artifactory_release : null - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { VAULT_LOG_LEVEL = var.vault_log_level } + install_dir = var.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = "raft" + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = "shamir" } } step "start_vault_agent" { module = "vault_agent" depends_on = [ - step.create_backend_cluster, step.build_vault, step.create_vault_cluster, ] @@ -159,8 +162,8 @@ scenario "agent" { } variables { - vault_instances = step.create_vault_cluster.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token vault_agent_template_destination = "/tmp/agent_output.txt" vault_agent_template_contents = "{{ with secret \\\"auth/token/lookup-self\\\" }}orphan={{ .Data.orphan }} display_name={{ .Data.display_name }}{{ end }}" } @@ -178,49 +181,64 @@ scenario "agent" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_agent_template_destination = "/tmp/agent_output.txt" vault_agent_expected_output = "orphan=true display_name=approle" } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awkms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name } - output "vault_cluster_priv_ips" { + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts + } + + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token } - output "vault_cluster_unseal_keys_b64" { - description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_unseal_keys_hex" { - description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 + } + + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex } } diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index bf683b67f7a5..86b284e1ecec 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -12,6 +12,12 @@ scenario "autopilot" { edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] artifact_type = ["package"] } + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } } terraform_cli = terraform_cli.default @@ -29,12 +35,17 @@ scenario "autopilot" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -105,36 +116,52 @@ scenario "autopilot" { } } - # This step creates a Vault cluster using a bundle downloaded from - # releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ - step.create_vpc, step.build_vault, + step.create_vault_cluster_targets ] + providers = { enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = "raft" + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + packages = local.packages + release = var.vault_autopilot_initial_release + storage_backend = "raft" storage_backend_addl_config = { autopilot_upgrade_version = var.vault_autopilot_initial_release.version } - unseal_method = matrix.seal - vault_install_dir = local.vault_install_dir - vault_release = var.vault_autopilot_initial_release - vault_license = step.read_license.license - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = matrix.seal } } @@ -152,12 +179,13 @@ scenario "autopilot" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } + step "verify_write_test_data" { module = module.vault_verify_write_data depends_on = [ @@ -172,9 +200,9 @@ scenario "autopilot" { variables { leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -186,8 +214,25 @@ scenario "autopilot" { } } - # This step creates a new Vault cluster using a bundle or package - # from the matrix.artifact_source, with the var.vault_product_version + step "create_vault_cluster_upgrade_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + cluster_name = step.create_vault_cluster_targets.cluster_name + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "upgrade_vault_cluster_with_autopilot" { module = module.vault_cluster depends_on = [ @@ -202,28 +247,25 @@ scenario "autopilot" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } + force_unseal = matrix.seal == "shamir" + initialize_cluster = false + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + root_token = step.create_vault_cluster.root_token + shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null storage_backend = "raft" storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config + storage_node_prefix = "upgrade_node" + target_hosts = step.create_vault_cluster_upgrade_targets.hosts unseal_method = matrix.seal - vault_cluster_tag = step.create_vault_cluster.vault_cluster_tag - vault_init = false - vault_install_dir = local.vault_install_dir - vault_license = step.read_license.license - vault_local_artifact_path = local.bundle_path - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_node_prefix = "upgrade_node" - vault_root_token = step.create_vault_cluster.vault_root_token - vault_unseal_when_no_init = matrix.seal == "shamir" - vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } } } @@ -231,6 +273,7 @@ scenario "autopilot" { module = module.vault_verify_unsealed depends_on = [ step.create_vault_cluster, + step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, ] @@ -240,7 +283,7 @@ scenario "autopilot" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances + vault_instances = step.create_vault_cluster_upgrade_targets.hosts } } @@ -257,14 +300,15 @@ scenario "autopilot" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.upgrade_vault_cluster_with_autopilot.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token } } step "verify_autopilot_await_server_removal_state" { module = module.vault_verify_autopilot depends_on = [ + step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, step.verify_raft_auto_join_voter ] @@ -277,8 +321,8 @@ scenario "autopilot" { vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_autopilot_upgrade_status = "await-server-removal" vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token } } @@ -286,6 +330,7 @@ scenario "autopilot" { module = module.vault_get_cluster_ips depends_on = [ step.create_vault_cluster, + step.create_vault_cluster_upgrade_targets, step.get_vault_cluster_ips, step.upgrade_vault_cluster_with_autopilot ] @@ -295,11 +340,11 @@ scenario "autopilot" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - added_vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token node_public_ip = step.get_vault_cluster_ips.leader_public_ip + added_vault_instances = step.create_vault_cluster_targets.hosts } } @@ -326,6 +371,7 @@ scenario "autopilot" { step "raft_remove_peers" { module = module.vault_raft_remove_peer depends_on = [ + step.create_vault_cluster_upgrade_targets, step.get_updated_vault_cluster_ips, step.upgrade_vault_cluster_with_autopilot, step.verify_autopilot_await_server_removal_state @@ -336,11 +382,11 @@ scenario "autopilot" { } variables { - vault_install_dir = local.vault_install_dir operator_instance = step.get_updated_vault_cluster_ips.leader_public_ip - remove_vault_instances = step.create_vault_cluster.vault_instances + remove_vault_instances = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir vault_instance_count = 3 - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -356,7 +402,7 @@ scenario "autopilot" { } variables { - old_vault_instances = step.create_vault_cluster.vault_instances + old_vault_instances = step.create_vault_cluster_targets.hosts vault_instance_count = 3 } } @@ -364,6 +410,7 @@ scenario "autopilot" { step "verify_autopilot_idle_state" { module = module.vault_verify_autopilot depends_on = [ + step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, step.verify_raft_auto_join_voter, step.remove_old_nodes @@ -377,15 +424,16 @@ scenario "autopilot" { vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_autopilot_upgrade_status = "idle" vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } step "verify_undo_logs_status" { - skip_step = try(semverconstraint(var.vault_product_version, "<1.13.0-0"), true) + skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0") module = module.vault_verify_undo_logs depends_on = [ + step.create_vault_cluster_upgrade_targets, step.remove_old_nodes, step.upgrade_vault_cluster_with_autopilot, step.verify_autopilot_idle_state @@ -397,78 +445,78 @@ scenario "autopilot" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name } - output "vault_cluster_priv_ips" { - description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "private_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.private_ips } - output "vault_cluster_root_token" { - description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_unseal_keys_b64" { - description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + output "root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.root_token } - output "vault_cluster_recovery_key_shares" { + output "recovery_key_shares" { description = "The Vault cluster recovery key shares" - value = step.create_vault_cluster.vault_recovery_key_shares + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_recovery_keys_b64" { + output "recovery_keys_b64" { description = "The Vault cluster recovery keys b64" - value = step.create_vault_cluster.vault_recovery_keys_b64 + value = step.create_vault_cluster.recovery_keys_b64 } - output "vault_cluster_recovery_keys_hex" { + output "recovery_keys_hex" { description = "The Vault cluster recovery keys hex" - value = step.create_vault_cluster.vault_recovery_keys_hex + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_hex" { - description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex } - output "upgraded_vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.upgrade_vault_cluster_with_autopilot.instance_ids + output "upgrade_hosts" { + description = "The Vault cluster target hosts" + value = step.upgrade_vault_cluster_with_autopilot.target_hosts } - output "upgraded_vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.upgrade_vault_cluster_with_autopilot.instance_public_ips + output "upgrade_private_ips" { + description = "The Vault cluster private IPs" + value = step.upgrade_vault_cluster_with_autopilot.private_ips } - output "upgraded_vault_cluster_priv_ips" { - description = "The Vault cluster private IPs" - value = step.upgrade_vault_cluster_with_autopilot.instance_private_ips + output "upgrade_public_ips" { + description = "The Vault cluster public IPs" + value = step.upgrade_vault_cluster_with_autopilot.public_ips } } diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl index a6c81672eae3..655bf0f55131 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-replication.hcl @@ -19,6 +19,12 @@ scenario "replication" { edition = ["ent.fips1402", "ent.hsm.fips1402"] artifact_type = ["package"] } + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } } terraform_cli = terraform_cli.default @@ -36,12 +42,17 @@ scenario "replication" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -131,37 +142,55 @@ scenario "replication" { } } - step "create_vault_primary_cluster" { + step "create_primary_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_primary_cluster" { module = module.vault_cluster depends_on = [ step.create_primary_backend_cluster, step.build_vault, + step.create_primary_cluster_targets ] + providers = { enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_primary_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag consul_release = matrix.primary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.primary_backend - unseal_method = matrix.primary_seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_license = step.read_license.license - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = matrix.primary_backend + target_hosts = step.create_primary_cluster_targets.hosts + unseal_method = matrix.primary_seal } } @@ -186,44 +215,62 @@ scenario "replication" { } } - step "create_vault_secondary_cluster" { + step "create_secondary_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_secondary_cluster" { module = module.vault_cluster depends_on = [ step.create_secondary_backend_cluster, step.build_vault, + step.create_secondary_cluster_targets ] + providers = { enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_secondary_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_secondary_backend_cluster.consul_cluster_tag consul_release = matrix.secondary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.secondary_backend - unseal_method = matrix.secondary_seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_license = step.read_license.license - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = matrix.secondary_backend + target_hosts = step.create_secondary_cluster_targets.hosts + unseal_method = matrix.secondary_seal } } - step "verify_vault_primary_unsealed" { + step "verify_that_vault_primary_cluster_is_unsealed" { module = module.vault_verify_unsealed depends_on = [ - step.create_vault_primary_cluster + step.create_primary_cluster ] providers = { @@ -231,15 +278,15 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } - step "verify_vault_secondary_unsealed" { + step "verify_that_vault_secondary_cluster_is_unsealed" { module = module.vault_verify_unsealed depends_on = [ - step.create_vault_secondary_cluster + step.create_secondary_cluster ] providers = { @@ -247,42 +294,42 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_secondary_cluster.vault_instances + vault_instances = step.create_secondary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } step "get_primary_cluster_ips" { module = module.vault_get_cluster_ips - depends_on = [step.verify_vault_primary_unsealed] + depends_on = [step.verify_that_vault_primary_cluster_is_unsealed] providers = { enos = local.enos_provider[matrix.distro] } variables { - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } step "get_secondary_cluster_ips" { module = module.vault_get_cluster_ips - depends_on = [step.verify_vault_secondary_unsealed] + depends_on = [step.verify_that_vault_secondary_cluster_is_unsealed] providers = { enos = local.enos_provider[matrix.distro] } variables { - vault_instances = step.create_vault_secondary_cluster.vault_instances + vault_instances = step.create_secondary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_secondary_cluster.vault_root_token + vault_root_token = step.create_secondary_cluster.root_token } } - step "verify_vault_primary_write_data" { + step "write_test_data_on_primary" { module = module.vault_verify_write_data depends_on = [step.get_primary_cluster_ips] @@ -294,9 +341,9 @@ scenario "replication" { variables { leader_public_ip = step.get_primary_cluster_ips.leader_public_ip leader_private_ip = step.get_primary_cluster_ips.leader_private_ip - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -304,7 +351,7 @@ scenario "replication" { module = module.vault_setup_perf_primary depends_on = [ step.get_primary_cluster_ips, - step.verify_vault_primary_write_data + step.write_test_data_on_primary ] providers = { @@ -315,7 +362,7 @@ scenario "replication" { primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip primary_leader_private_ip = step.get_primary_cluster_ips.leader_private_ip vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -330,7 +377,7 @@ scenario "replication" { variables { primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -346,7 +393,7 @@ scenario "replication" { secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip secondary_leader_private_ip = step.get_secondary_cluster_ips.leader_private_ip vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_secondary_cluster.vault_root_token + vault_root_token = step.create_secondary_cluster.root_token wrapping_token = step.generate_secondary_token.secondary_token } } @@ -356,8 +403,8 @@ scenario "replication" { step "unseal_secondary_followers" { module = module.vault_unseal_nodes depends_on = [ - step.create_vault_primary_cluster, - step.create_vault_secondary_cluster, + step.create_primary_cluster, + step.create_secondary_cluster, step.get_secondary_cluster_ips, step.configure_performance_replication_secondary ] @@ -369,12 +416,12 @@ scenario "replication" { variables { follower_public_ips = step.get_secondary_cluster_ips.follower_public_ips vault_install_dir = local.vault_install_dir - vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : step.create_vault_primary_cluster.vault_recovery_keys_hex + vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : step.create_primary_cluster.recovery_keys_hex vault_seal_type = matrix.primary_seal == "shamir" ? matrix.primary_seal : matrix.secondary_seal } } - step "verify_vault_secondary_unsealed_after_replication" { + step "verify_secondary_cluster_is_unsealed_after_enabling_replication" { module = module.vault_verify_unsealed depends_on = [ step.unseal_secondary_followers @@ -385,14 +432,14 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_secondary_cluster.vault_instances + vault_instances = step.create_secondary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } step "verify_performance_replication" { module = module.vault_verify_performance_replication - depends_on = [step.verify_vault_secondary_unsealed_after_replication] + depends_on = [step.verify_secondary_cluster_is_unsealed_after_enabling_replication] providers = { enos = local.enos_provider[matrix.distro] @@ -412,7 +459,7 @@ scenario "replication" { depends_on = [ step.verify_performance_replication, step.get_secondary_cluster_ips, - step.verify_vault_primary_write_data + step.write_test_data_on_primary ] providers = { @@ -425,13 +472,32 @@ scenario "replication" { } } - step "add_primary_cluster_nodes" { + step "create_more_primary_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + + step "add_more_nodes_to_primary_cluster" { module = module.vault_cluster depends_on = [ step.create_vpc, step.create_primary_backend_cluster, - step.create_vault_primary_cluster, - step.verify_replicated_data + step.create_primary_cluster, + step.verify_replicated_data, + step.create_more_primary_cluster_targets ] providers = { @@ -439,45 +505,42 @@ scenario "replication" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_primary_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag consul_release = matrix.primary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.primary_backend - unseal_method = matrix.primary_seal - vault_cluster_tag = step.create_vault_primary_cluster.vault_cluster_tag - vault_init = false - vault_license = step.read_license.license - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_node_prefix = "newprimary_node" - vault_root_token = step.create_vault_primary_cluster.vault_root_token - vault_unseal_when_no_init = matrix.primary_seal == "shamir" - vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } - } - } - - step "verify_add_node_unsealed" { + force_unseal = matrix.primary_seal == "shamir" + initialize_cluster = false + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + root_token = step.create_primary_cluster.root_token + shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null + storage_backend = matrix.primary_backend + storage_node_prefix = "newprimary_node" + target_hosts = step.create_more_primary_cluster_targets.hosts + unseal_method = matrix.primary_seal + } + } + + step "verify_more_primary_nodes_unsealed" { module = module.vault_verify_unsealed - depends_on = [step.add_primary_cluster_nodes] + depends_on = [step.add_more_nodes_to_primary_cluster] providers = { enos = local.enos_provider[matrix.distro] } variables { - vault_instances = step.add_primary_cluster_nodes.vault_instances + vault_instances = step.create_more_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } @@ -486,9 +549,9 @@ scenario "replication" { skip_step = matrix.primary_backend != "raft" module = module.vault_verify_raft_auto_join_voter depends_on = [ - step.add_primary_cluster_nodes, - step.create_vault_primary_cluster, - step.verify_add_node_unsealed + step.add_more_nodes_to_primary_cluster, + step.create_primary_cluster, + step.verify_more_primary_nodes_unsealed ] providers = { @@ -496,9 +559,9 @@ scenario "replication" { } variables { - vault_instances = step.add_primary_cluster_nodes.vault_instances + vault_instances = step.create_more_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -506,7 +569,7 @@ scenario "replication" { module = module.shutdown_node depends_on = [ step.get_primary_cluster_ips, - step.verify_add_node_unsealed + step.verify_more_primary_nodes_unsealed ] providers = { @@ -537,7 +600,7 @@ scenario "replication" { step "get_updated_primary_cluster_ips" { module = module.vault_get_cluster_ips depends_on = [ - step.add_primary_cluster_nodes, + step.add_more_nodes_to_primary_cluster, step.remove_primary_follower_1, step.remove_primary_leader ] @@ -547,10 +610,10 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - added_vault_instances = step.add_primary_cluster_nodes.vault_instances - vault_root_token = step.create_vault_primary_cluster.vault_root_token + added_vault_instances = step.create_more_primary_cluster_targets.hosts + vault_root_token = step.create_primary_cluster.root_token node_public_ip = step.get_primary_cluster_ips.follower_public_ip_2 } } @@ -572,112 +635,97 @@ scenario "replication" { } } - output "vault_primary_cluster_pub_ips" { - description = "The Vault primary cluster public IPs" - value = step.create_vault_primary_cluster.instance_public_ips + output "primary_cluster_hosts" { + description = "The Vault primary cluster target hosts" + value = step.create_primary_cluster_targets.hosts } - output "vault_primary_cluster_priv_ips" { - description = "The Vault primary cluster private IPs" - value = step.create_vault_primary_cluster.instance_private_ips + output "primary_cluster_additional_hosts" { + description = "The Vault added new node on primary cluster target hosts" + value = step.create_more_primary_cluster_targets.hosts } - output "vault_primary_newnode_pub_ip" { - description = "The Vault added new node on primary cluster public IP" - value = step.add_primary_cluster_nodes.instance_public_ips - } - - output "vault_primary_newnode_priv_ip" { - description = "The Vault added new node on primary cluster private IP" - value = step.add_primary_cluster_nodes.instance_private_ips - } - - output "vault_primary_cluster_root_token" { + output "primary_cluster_root_token" { description = "The Vault primary cluster root token" - value = step.create_vault_primary_cluster.vault_root_token + value = step.create_primary_cluster.root_token } - output "vault_primary_cluster_unseal_keys_b64" { + output "primary_cluster_unseal_keys_b64" { description = "The Vault primary cluster unseal keys" - value = step.create_vault_primary_cluster.vault_unseal_keys_b64 + value = step.create_primary_cluster.unseal_keys_b64 } - output "vault_primary_cluster_unseal_keys_hex" { + output "primary_cluster_unseal_keys_hex" { description = "The Vault primary cluster unseal keys hex" - value = step.create_vault_primary_cluster.vault_unseal_keys_hex + value = step.create_primary_cluster.unseal_keys_hex } - output "vault_primary_cluster_recovery_key_shares" { + output "primary_cluster_recovery_key_shares" { description = "The Vault primary cluster recovery key shares" - value = step.create_vault_primary_cluster.vault_recovery_key_shares + value = step.create_primary_cluster.recovery_key_shares } - output "vault_primary_cluster_recovery_keys_b64" { + output "primary_cluster_recovery_keys_b64" { description = "The Vault primary cluster recovery keys b64" - value = step.create_vault_primary_cluster.vault_recovery_keys_b64 + value = step.create_primary_cluster.recovery_keys_b64 } - output "vault_primary_cluster_recovery_keys_hex" { + output "primary_cluster_recovery_keys_hex" { description = "The Vault primary cluster recovery keys hex" - value = step.create_vault_primary_cluster.vault_recovery_keys_hex + value = step.create_primary_cluster.recovery_keys_hex } - output "vault_secondary_cluster_pub_ips" { + output "secondary_cluster_hosts" { description = "The Vault secondary cluster public IPs" - value = step.create_vault_secondary_cluster.instance_public_ips - } - - output "vault_secondary_cluster_priv_ips" { - description = "The Vault secondary cluster private IPs" - value = step.create_vault_secondary_cluster.instance_private_ips + value = step.create_secondary_cluster_targets.hosts } - output "vault_primary_performance_replication_status" { + output "initial_primary_replication_status" { description = "The Vault primary cluster performance replication status" value = step.verify_performance_replication.primary_replication_status } - output "vault_replication_known_primary_cluster_addrs" { + output "initial_known_primary_cluster_addresses" { description = "The Vault secondary cluster performance replication status" value = step.verify_performance_replication.known_primary_cluster_addrs } - output "vault_secondary_performance_replication_status" { + output "initial_secondary_performance_replication_status" { description = "The Vault secondary cluster performance replication status" value = step.verify_performance_replication.secondary_replication_status } - output "vault_primary_updated_performance_replication_status" { + output "intial_primary_replication_data_secondaries" { + description = "The Vault primary cluster secondaries connection status" + value = step.verify_performance_replication.primary_replication_data_secondaries + } + + output "initial_secondary_replication_data_primaries" { + description = "The Vault secondary cluster primaries connection status" + value = step.verify_performance_replication.secondary_replication_data_primaries + } + + output "updated_primary_replication_status" { description = "The Vault updated primary cluster performance replication status" value = step.verify_updated_performance_replication.primary_replication_status } - output "vault_updated_replication_known_primary_cluster_addrs" { + output "updated_known_primary_cluster_addresses" { description = "The Vault secondary cluster performance replication status" value = step.verify_updated_performance_replication.known_primary_cluster_addrs } - output "verify_secondary_updated_performance_replication_status" { + output "updated_secondary_replication_status" { description = "The Vault updated secondary cluster performance replication status" value = step.verify_updated_performance_replication.secondary_replication_status } - output "primary_replication_data_secondaries" { - description = "The Vault primary cluster secondaries connection status" - value = step.verify_performance_replication.primary_replication_data_secondaries - } - - output "secondary_replication_data_primaries" { - description = "The Vault secondary cluster primaries connection status" - value = step.verify_performance_replication.secondary_replication_data_primaries - } - - output "primary_updated_replication_data_secondaries" { + output "updated_primary_replication_data_secondaries" { description = "The Vault updated primary cluster secondaries connection status" value = step.verify_updated_performance_replication.primary_replication_data_secondaries } - output "secondary_updated_replication_data_primaries" { + output "updated_secondary_replication_data_primaries" { description = "The Vault updated secondary cluster primaries connection status" value = step.verify_updated_performance_replication.secondary_replication_data_primaries } diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 717f275ffa73..9055b8cc12df 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -14,6 +14,12 @@ scenario "smoke" { edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] artifact_type = ["package"] } + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } } terraform_cli = terraform_cli.default @@ -32,12 +38,17 @@ scenario "smoke" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -134,11 +145,30 @@ scenario "smoke" { } } + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -146,26 +176,24 @@ scenario "smoke" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag consul_release = matrix.backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.backend - unseal_method = matrix.seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = matrix.seal } } @@ -178,9 +206,9 @@ scenario "smoke" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -193,13 +221,13 @@ scenario "smoke" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_edition = matrix.edition vault_install_dir = local.vault_install_dir vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -213,7 +241,7 @@ scenario "smoke" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts } } @@ -231,9 +259,9 @@ scenario "smoke" { variables { leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -248,8 +276,8 @@ scenario "smoke" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } @@ -264,7 +292,7 @@ scenario "smoke" { variables { vault_edition = matrix.edition vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts } } @@ -294,63 +322,63 @@ scenario "smoke" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts } - output "vault_cluster_priv_ips" { + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token } - output "vault_cluster_recovery_key_shares" { + output "recovery_key_shares" { description = "The Vault cluster recovery key shares" - value = step.create_vault_cluster.vault_recovery_key_shares + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_recovery_keys_b64" { + output "recovery_keys_b64" { description = "The Vault cluster recovery keys b64" - value = step.create_vault_cluster.vault_recovery_keys_b64 + value = step.create_vault_cluster.recovery_keys_b64 } - output "vault_cluster_recovery_keys_hex" { + output "recovery_keys_hex" { description = "The Vault cluster recovery keys hex" - value = step.create_vault_cluster.vault_recovery_keys_hex + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_b64" { + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_unseal_keys_hex" { + output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex - } - - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + value = step.create_vault_cluster.unseal_keys_hex } } diff --git a/enos/enos-scenario-ui.hcl b/enos/enos-scenario-ui.hcl index 77335e6ad8f8..b1f56582702c 100644 --- a/enos/enos-scenario-ui.hcl +++ b/enos/enos-scenario-ui.hcl @@ -110,11 +110,29 @@ scenario "ui" { } } + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.create_vpc.ami_ids[local.distro][local.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -122,20 +140,22 @@ scenario "ui" { } variables { - ami_id = step.create_vpc.ami_ids[local.distro][local.arch] - common_tags = local.tags - consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.backend - unseal_method = local.seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { VAULT_LOG_LEVEL = var.vault_log_level } + consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag + consul_release = matrix.backend == "consul" ? { + edition = var.backend_edition + version = local.consul_version + } : null + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = local.seal } } @@ -143,52 +163,72 @@ scenario "ui" { module = module.vault_test_ui variables { - vault_addr = step.create_vault_cluster.instance_public_ips[0] - vault_root_token = step.create_vault_cluster.vault_root_token - vault_unseal_keys = step.create_vault_cluster.vault_recovery_keys_b64 - vault_recovery_threshold = step.create_vault_cluster.vault_recovery_threshold + vault_addr = step.create_vault_cluster_targets.hosts[0].public_ip + vault_root_token = step.create_vault_cluster.root_token + vault_unseal_keys = step.create_vault_cluster.recovery_keys_b64 + vault_recovery_threshold = step.create_vault_cluster.recovery_threshold ui_test_filter = local.ui_test_filter } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts } - output "vault_cluster_priv_ips" { + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token + } + + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares + } + + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_b64" { + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_unseal_keys_hex" { + output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex + value = step.create_vault_cluster.unseal_keys_hex } - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + output "ui_test_environment" { + value = step.test_ui.ui_test_environment + description = "The environment variables that are required in order to run the test:enos yarn target" } output "ui_test_stderr" { @@ -200,9 +240,4 @@ scenario "ui" { description = "The stdout of the ui tests that ran" value = step.test_ui.ui_test_stdout } - - output "ui_test_environment" { - value = step.test_ui.ui_test_environment - description = "The environment variables that are required in order to run the test:enos yarn target" - } } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 5a188695c5e4..cb77956bd4f5 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -32,12 +32,17 @@ scenario "upgrade" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -135,13 +140,30 @@ scenario "upgrade" { } } - # This step creates a Vault cluster using a bundle downloaded from - # releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -149,25 +171,24 @@ scenario "upgrade" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag consul_release = matrix.backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.backend - unseal_method = matrix.seal - vault_install_dir = local.vault_install_dir - vault_release = var.vault_upgrade_initial_release - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + packages = local.packages + release = var.vault_upgrade_initial_release + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = matrix.seal } } @@ -180,9 +201,9 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -200,9 +221,9 @@ scenario "upgrade" { variables { leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -220,11 +241,11 @@ scenario "upgrade" { variables { vault_api_addr = "http://localhost:8200" - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_local_artifact_path = local.bundle_path vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null vault_install_dir = local.vault_install_dir - vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null + vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null vault_seal_type = matrix.seal } } @@ -241,13 +262,13 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_edition = matrix.edition vault_install_dir = local.vault_install_dir vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -263,9 +284,9 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -282,7 +303,7 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } @@ -319,63 +340,63 @@ scenario "upgrade" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name } - output "vault_cluster_priv_ips" { + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts + } + + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token } - output "vault_cluster_recovery_key_shares" { + output "recovery_key_shares" { description = "The Vault cluster recovery key shares" - value = step.create_vault_cluster.vault_recovery_key_shares + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_recovery_keys_b64" { + output "recovery_keys_b64" { description = "The Vault cluster recovery keys b64" - value = step.create_vault_cluster.vault_recovery_keys_b64 + value = step.create_vault_cluster.recovery_keys_b64 } - output "vault_cluster_recovery_keys_hex" { + output "recovery_keys_hex" { description = "The Vault cluster recovery keys hex" - value = step.create_vault_cluster.vault_recovery_keys_hex + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_b64" { + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_unseal_keys_hex" { + output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex - } - - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + value = step.create_vault_cluster.unseal_keys_hex } } diff --git a/enos/modules/target_ec2_instances/main.tf b/enos/modules/target_ec2_instances/main.tf new file mode 100644 index 000000000000..26a520d3e8ce --- /dev/null +++ b/enos/modules/target_ec2_instances/main.tf @@ -0,0 +1,181 @@ +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.3.24" + } + } +} + +data "aws_vpc" "vpc" { + id = var.vpc_id +} + +data "aws_subnets" "vpc" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +data "aws_kms_key" "kms_key" { + key_id = var.awskms_unseal_key_arn +} + +data "aws_iam_policy_document" "target" { + statement { + resources = ["*"] + + actions = [ + "ec2:DescribeInstances", + "secretsmanager:*" + ] + } + + statement { + resources = [var.awskms_unseal_key_arn] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } +} + +data "aws_iam_policy_document" "target_instance_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "enos_environment" "localhost" {} + +resource "random_string" "cluster_name" { + length = 8 + lower = true + upper = false + numeric = false + special = false +} + +locals { + instances = toset([for idx in range(var.instance_count) : tostring(idx)]) + cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result) + name_prefix = "${var.project_name}-${local.cluster_name}" +} + +resource "aws_iam_role" "target_instance_role" { + name = "target_instance_role-${random_string.cluster_name.result}" + assume_role_policy = data.aws_iam_policy_document.target_instance_role.json +} + +resource "aws_iam_instance_profile" "target" { + name = "${local.name_prefix}-target" + role = aws_iam_role.target_instance_role.name +} + +resource "aws_iam_role_policy" "target" { + name = "${local.name_prefix}-target" + role = aws_iam_role.target_instance_role.id + policy = data.aws_iam_policy_document.target.json +} + +resource "aws_security_group" "target" { + name = "${local.name_prefix}-target" + description = "Target instance security group" + vpc_id = var.vpc_id + + # SSH traffic + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) + } + + # Vault traffic + ingress { + from_port = 8200 + to_port = 8201 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + formatlist("%s/32", var.ssh_allow_ips) + ]) + } + + # Consul traffic + ingress { + from_port = 8301 + to_port = 8301 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) + } + + ingress { + from_port = 8301 + to_port = 8301 + protocol = "udp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) + } + + # Internal traffic + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + } + + # External traffic + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = merge( + var.common_tags, + { + Name = "${local.name_prefix}-sg" + }, + ) +} + +resource "aws_instance" "targets" { + for_each = local.instances + ami = var.ami_id + instance_type = var.instance_type + vpc_security_group_ids = [aws_security_group.target.id] + subnet_id = tolist(data.aws_subnets.vpc.ids)[each.key % length(data.aws_subnets.vpc.ids)] + key_name = var.ssh_keypair + iam_instance_profile = aws_iam_instance_profile.target.name + tags = merge( + var.common_tags, + { + Name = "${local.name_prefix}-target-instance" + Type = local.cluster_name + }, + ) +} diff --git a/enos/modules/target_ec2_instances/outputs.tf b/enos/modules/target_ec2_instances/outputs.tf new file mode 100644 index 000000000000..9428bfdb9915 --- /dev/null +++ b/enos/modules/target_ec2_instances/outputs.tf @@ -0,0 +1,11 @@ +output "cluster_name" { + value = local.cluster_name +} + +output "hosts" { + description = "The ec2 instance target hosts" + value = { for idx in range(var.instance_count) : idx => { + public_ip = aws_instance.targets[idx].public_ip + private_ip = aws_instance.targets[idx].private_ip + } } +} diff --git a/enos/modules/target_ec2_instances/variables.tf b/enos/modules/target_ec2_instances/variables.tf new file mode 100644 index 000000000000..89dbbf03c776 --- /dev/null +++ b/enos/modules/target_ec2_instances/variables.tf @@ -0,0 +1,61 @@ +variable "ami_id" { + description = "The machine image identifier" + type = string +} + +variable "awskms_unseal_key_arn" { + type = string + description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" + default = null +} + +variable "cluster_name" { + type = string + description = "A unique cluster identifier" + default = null +} + +variable "common_tags" { + description = "Common tags for cloud resources" + type = map(string) + default = { "Project" : "Enos" } +} + +variable "instance_count" { + description = "The number of target instances to create" + type = number + default = 3 +} + +variable "instance_type" { + description = "The instance machine type" + type = string + default = "t3.small" +} + +variable "project_name" { + description = "A unique project name" + type = string +} + +variable "spot_price_max" { + description = "Unused shim variable to match target_ec2_spot_fleet" + type = string + default = null +} + +variable "ssh_allow_ips" { + description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" + type = list(string) + default = [] +} + +variable "ssh_keypair" { + description = "SSH keypair used to connect to EC2 instances" + type = string +} + +variable "vpc_id" { + description = "The identifier of the VPC where the target instances will be created" + type = string +} diff --git a/enos/modules/target_ec2_spot_fleet/main.tf b/enos/modules/target_ec2_spot_fleet/main.tf new file mode 100644 index 000000000000..4e55da2dd095 --- /dev/null +++ b/enos/modules/target_ec2_spot_fleet/main.tf @@ -0,0 +1,388 @@ +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.3.24" + } + } +} + +data "aws_vpc" "vpc" { + id = var.vpc_id +} + +data "aws_subnets" "vpc" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +data "aws_kms_key" "kms_key" { + key_id = var.awskms_unseal_key_arn +} + +data "aws_iam_policy_document" "target" { + statement { + resources = ["*"] + + actions = [ + "ec2:DescribeInstances", + "secretsmanager:*" + ] + } + + statement { + resources = [var.awskms_unseal_key_arn] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } +} + +data "aws_iam_policy_document" "target_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "fleet" { + statement { + resources = ["*"] + + actions = [ + "ec2:DescribeImages", + "ec2:DescribeSubnets", + "ec2:RequestSpotInstances", + "ec2:TerminateInstances", + "ec2:DescribeInstanceStatus", + "ec2:CancelSpotFleetRequests", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:StartInstances", + "ec2:StopInstances", + ] + } + + statement { + effect = "Deny" + + resources = [ + "arn:aws:ec2:*:*:instance/*", + ] + + actions = [ + "ec2:RunInstances", + ] + + condition { + test = "StringNotEquals" + variable = "ec2:InstanceMarketType" + values = ["spot"] + } + } + + statement { + resources = ["*"] + + actions = [ + "iam:PassRole", + ] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = [ + "ec2.amazonaws.com", + ] + } + } + + statement { + resources = [ + "arn:aws:elasticloadbalancing:*:*:loadbalancer/*", + ] + + actions = [ + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + ] + } + + statement { + resources = [ + "arn:aws:elasticloadbalancing:*:*:*/*" + ] + + actions = [ + "elasticloadbalancing:RegisterTargets" + ] + } +} + +data "aws_iam_policy_document" "fleet_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["spotfleet.amazonaws.com"] + } + } +} + +data "enos_environment" "localhost" {} + +resource "random_string" "cluster_name" { + length = 8 + lower = true + upper = false + numeric = false + special = false +} + +resource "random_string" "unique_id" { + length = 4 + lower = true + upper = false + numeric = false + special = false +} + +locals { + instances = toset([for idx in range(var.instance_count) : tostring(idx)]) + cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result) + name_prefix = "${var.project_name}-${local.cluster_name}-${random_string.unique_id.result}" + fleet_tag = "${local.name_prefix}-spot-fleet-target" + fleet_tags = { + Name = "${local.name_prefix}-target" + Type = local.cluster_name + SpotFleet = local.fleet_tag + } +} + +resource "aws_iam_role" "target" { + name = "${local.name_prefix}-target-role" + assume_role_policy = data.aws_iam_policy_document.target_role.json +} + +resource "aws_iam_instance_profile" "target" { + name = "${local.name_prefix}-target-profile" + role = aws_iam_role.target.name +} + +resource "aws_iam_role_policy" "target" { + name = "${local.name_prefix}-target-policy" + role = aws_iam_role.target.id + policy = data.aws_iam_policy_document.target.json +} + +resource "aws_iam_role" "fleet" { + name = "${local.name_prefix}-fleet-role" + assume_role_policy = data.aws_iam_policy_document.fleet_role.json +} + +resource "aws_iam_role_policy" "fleet" { + name = "${local.name_prefix}-fleet-policy" + role = aws_iam_role.fleet.id + policy = data.aws_iam_policy_document.fleet.json +} + +resource "aws_security_group" "target" { + name = "${local.name_prefix}-target" + description = "Target instance security group" + vpc_id = var.vpc_id + + # SSH traffic + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) + } + + # Vault traffic + ingress { + from_port = 8200 + to_port = 8201 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + formatlist("%s/32", var.ssh_allow_ips) + ]) + } + + # Consul traffic + ingress { + from_port = 8301 + to_port = 8301 + protocol = "tcp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) + } + + ingress { + from_port = 8301 + to_port = 8301 + protocol = "udp" + cidr_blocks = flatten([ + formatlist("%s/32", data.enos_environment.localhost.public_ip_addresses), + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ]) + } + + # Internal traffic + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + } + + # External traffic + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = merge( + var.common_tags, + { + Name = "${local.name_prefix}-sg" + }, + ) +} + +resource "aws_launch_template" "target" { + name = "${local.name_prefix}-target" + image_id = var.ami_id + key_name = var.ssh_keypair + + iam_instance_profile { + name = aws_iam_instance_profile.target.name + } + + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.target.id] + } + + tag_specifications { + resource_type = "instance" + + tags = merge( + var.common_tags, + local.fleet_tags, + ) + } +} + +# There are three primary knobs we can turn to try and optimize our costs by +# using a spot fleet: our min and max instance requirements, our max bid +# price, and the allocation strategy to use when fulfilling the spot request. +# We've currently configured our instance requirements to allow for anywhere +# from 2-4 vCPUs and 4-16GB of RAM. We intentionally have a wide range +# to allow for a large instance size pool to be considered. Our next knob is our +# max bid price. As we're using spot fleets to save on instance cost, we never +# want to pay more for an instance than we were on-demand. We've set the max price +# to equal what we pay for t3.medium instances on-demand, which are the smallest +# reliable size for Vault scenarios. The final knob is the allocation strategy +# that AWS will use when looking for instances that meet our resource and cost +# requirements. We're using the "lowestPrice" strategy to get the absolute +# cheapest machines that will fit the requirements, but it comes with a slightly +# higher capacity risk than say, "capacityOptimized" or "priceCapacityOptimized". +# Unless we see capacity issues or instances being shut down then we ought to +# stick with that strategy. +resource "aws_spot_fleet_request" "targets" { + allocation_strategy = "lowestPrice" + fleet_type = "request" + iam_fleet_role = aws_iam_role.fleet.arn + // Set this to zero so re-runs don't plan for replacement + instance_pools_to_use_count = 0 + target_capacity = var.instance_count + terminate_instances_on_delete = true + wait_for_fulfillment = true + + launch_template_config { + launch_template_specification { + id = aws_launch_template.target.id + version = aws_launch_template.target.latest_version + } + + overrides { + spot_price = var.spot_price_max + subnet_id = data.aws_subnets.vpc.ids[0] + + instance_requirements { + burstable_performance = "included" + + memory_mib { + min = var.instance_mem_min + max = var.instance_mem_max + } + + vcpu_count { + min = var.instance_cpu_min + max = var.instance_cpu_max + } + } + } + } + + tags = merge( + var.common_tags, + local.fleet_tags, + ) +} + +data "aws_instances" "targets" { + depends_on = [ + aws_spot_fleet_request.targets, + ] + + instance_tags = local.fleet_tags + instance_state_names = [ + "pending", + "running", + ] + + filter { + name = "image-id" + values = [var.ami_id] + } + + filter { + name = "iam-instance-profile.arn" + values = [aws_iam_instance_profile.target.arn] + } +} + +data "aws_instance" "targets" { + depends_on = [ + aws_spot_fleet_request.targets, + data.aws_instances.targets + ] + for_each = local.instances + + instance_id = data.aws_instances.targets.ids[each.key] +} diff --git a/enos/modules/target_ec2_spot_fleet/outputs.tf b/enos/modules/target_ec2_spot_fleet/outputs.tf new file mode 100644 index 000000000000..2248388da521 --- /dev/null +++ b/enos/modules/target_ec2_spot_fleet/outputs.tf @@ -0,0 +1,11 @@ +output "cluster_name" { + value = local.cluster_name +} + +output "hosts" { + description = "The spot fleet target hosts" + value = { for idx in range(var.instance_count) : idx => { + public_ip = data.aws_instance.targets[idx].public_ip + private_ip = data.aws_instance.targets[idx].private_ip + } } +} diff --git a/enos/modules/target_ec2_spot_fleet/variables.tf b/enos/modules/target_ec2_spot_fleet/variables.tf new file mode 100644 index 000000000000..da41866554f8 --- /dev/null +++ b/enos/modules/target_ec2_spot_fleet/variables.tf @@ -0,0 +1,88 @@ +variable "ami_id" { + description = "The machine image identifier" + type = string +} + +variable "awskms_unseal_key_arn" { + type = string + description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" + default = null +} + +variable "cluster_name" { + type = string + description = "A unique cluster identifier" + default = null +} + +variable "common_tags" { + description = "Common tags for cloud resources" + type = map(string) + default = { + Project = "Vault" + } +} + +variable "instance_mem_min" { + description = "The minimum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)" + type = number + default = 4096 // ~4 GB +} + +variable "instance_mem_max" { + description = "The maximum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)" + type = number + default = 16385 // ~16 GB +} + +variable "instance_cpu_min" { + description = "The minimum number of vCPU's for each instance in the fleet" + type = number + default = 2 +} + +variable "instance_cpu_max" { + description = "The maximum number of vCPU's for each instance in the fleet" + type = number + default = 8 // Unlikely we'll ever get that high due to spot price bid protection +} + +variable "instance_count" { + description = "The number of target instances to create" + type = number + default = 3 +} + +variable "instance_type" { + description = "Shim variable for target module variable compatibility that is not used. The spot fleet determines instance sizes" + type = string + default = null +} + +variable "project_name" { + description = "A unique project name" + type = string +} + +variable "spot_price_max" { + description = "The maximum hourly price to pay for each target instance" + type = string + // Current on-demand cost of linux t3.medium in us-east. + default = "0.0416" +} + +variable "ssh_allow_ips" { + description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" + type = list(string) + default = [] +} + +variable "ssh_keypair" { + description = "SSH keypair used to connect to EC2 instances" + type = string +} + +variable "vpc_id" { + description = "The identifier of the VPC where the target instances will be created" + type = string +} diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf new file mode 100644 index 000000000000..08455ca8f648 --- /dev/null +++ b/enos/modules/vault_cluster/main.tf @@ -0,0 +1,335 @@ +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.3.2" + } + } +} + +data "enos_environment" "localhost" {} + +locals { + bin_path = "${var.install_dir}/vault" + consul_bin_path = "${var.consul_install_dir}/consul" + key_shares = { + "awskms" = null + "shamir" = 5 + } + key_threshold = { + "awskms" = null + "shamir" = 3 + } + // In order to get Terraform to plan we have to use collections with keys + // that are known at plan time. In order for our module to work our var.target_hosts + // must be a map with known keys at plan time. Here we're creating locals + // that keep track of index values that point to our target hosts. + followers = toset(slice(local.instances, 1, length(local.instances))) + instances = [for idx in range(length(var.target_hosts)) : tostring(idx)] + leader = toset(slice(local.instances, 0, 1)) + recovery_shares = { + "awskms" = 5 + "shamir" = null + } + recovery_threshold = { + "awskms" = 3 + "shamir" = null + } + seal = { + "awskms" = { + type = "awskms" + attributes = { + kms_key_id = var.awskms_unseal_key_arn + } + } + "shamir" = { + type = "shamir" + attributes = null + } + } + storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ? + merge( + { + node_id = "${var.storage_node_prefix}_${idx}" + }, + var.storage_backend_addl_config + ) : + { + address = "127.0.0.1:8500" + path = "vault" + }) + ] +} + +resource "enos_remote_exec" "install_packages" { + for_each = { + for idx, host in var.target_hosts : idx => var.target_hosts[idx] + if length(var.packages) > 0 + } + + content = templatefile("${path.module}/templates/install-packages.sh", { + packages = join(" ", var.packages) + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_bundle_install" "consul" { + for_each = { + for idx, host in var.target_hosts : idx => var.target_hosts[idx] + if var.storage_backend == "consul" + } + + destination = var.consul_install_dir + release = merge(var.consul_release, { product = "consul" }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_bundle_install" "vault" { + for_each = var.target_hosts + + destination = var.install_dir + release = var.release == null ? var.release : merge({ product = "vault" }, var.release) + artifactory = var.artifactory_release + path = var.local_artifact_path + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_consul_start" "consul" { + for_each = enos_bundle_install.consul + + bin_path = local.consul_bin_path + data_dir = var.consul_data_dir + config = { + data_dir = var.consul_data_dir + datacenter = "dc1" + retry_join = ["provider=aws tag_key=Type tag_value=${var.consul_cluster_tag}"] + server = false + bootstrap_expect = 0 + log_level = "INFO" + log_file = var.consul_log_file + } + unit_name = "consul" + username = "consul" + + transport = { + ssh = { + host = var.target_hosts[each.key].public_ip + } + } +} + +resource "enos_vault_start" "leader" { + depends_on = [ + enos_consul_start.consul, + enos_bundle_install.vault, + ] + for_each = local.leader + + bin_path = local.bin_path + config_dir = var.config_dir + environment = var.config_env_vars + config = { + api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" + cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" + cluster_name = var.cluster_name + listener = { + type = "tcp" + attributes = { + address = "0.0.0.0:8200" + tls_disable = "true" + } + } + storage = { + type = var.storage_backend + attributes = ({ for key, value in local.storage_config[each.key] : key => value }) + } + seal = local.seal[var.unseal_method] + ui = true + } + license = var.license + manage_service = var.manage_service + username = "vault" + unit_name = "vault" + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_start" "followers" { + depends_on = [ + enos_vault_start.leader, + ] + for_each = local.followers + + bin_path = local.bin_path + config_dir = var.config_dir + environment = var.config_env_vars + config = { + api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" + cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" + cluster_name = var.cluster_name + listener = { + type = "tcp" + attributes = { + address = "0.0.0.0:8200" + tls_disable = "true" + } + } + storage = { + type = var.storage_backend + attributes = { for key, value in local.storage_config[each.key] : key => value } + } + seal = local.seal[var.unseal_method] + ui = true + } + license = var.license + manage_service = var.manage_service + username = "vault" + unit_name = "vault" + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_init" "leader" { + depends_on = [ + enos_vault_start.followers, + ] + for_each = toset([ + for idx, leader in local.leader : leader + if var.initialize_cluster + ]) + + bin_path = local.bin_path + vault_addr = enos_vault_start.leader[0].config.api_addr + + key_shares = local.key_shares[var.unseal_method] + key_threshold = local.key_threshold[var.unseal_method] + + recovery_shares = local.recovery_shares[var.unseal_method] + recovery_threshold = local.recovery_threshold[var.unseal_method] + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_unseal" "leader" { + depends_on = [ + enos_vault_start.followers, + enos_vault_init.leader, + ] + for_each = enos_vault_init.leader // only unseal the leader if we initialized it + + bin_path = local.bin_path + vault_addr = enos_vault_start.leader[each.key].config.api_addr + seal_type = var.unseal_method + unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) + + transport = { + ssh = { + host = var.target_hosts[tolist(local.leader)[0]].public_ip + } + } +} + +resource "enos_vault_unseal" "followers" { + depends_on = [ + enos_vault_init.leader, + enos_vault_unseal.leader, + ] + // Only unseal followers if we're not using an auto-unseal method and we've + // initialized the cluster + for_each = toset([ + for idx, follower in local.followers : follower + if var.unseal_method == "shamir" && var.initialize_cluster + ]) + + bin_path = local.bin_path + vault_addr = enos_vault_start.followers[each.key].config.api_addr + seal_type = var.unseal_method + unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +// Force unseal the cluster. This is used if the vault-cluster module is used +// to add additional nodes to a cluster via auto-pilot, or some other means. +// When that happens we'll want to set initialize_cluster to false and +// force_unseal to true. +resource "enos_vault_unseal" "maybe_force_unseal" { + depends_on = [ + enos_vault_start.followers, + ] + for_each = { + for idx, host in var.target_hosts : idx => host + if var.force_unseal && !var.initialize_cluster + } + + bin_path = local.bin_path + vault_addr = "http://localhost:8200" + seal_type = var.unseal_method + unseal_keys = coalesce( + var.shamir_unseal_keys, + try(enos_vault_init.leader[0].unseal_keys_hex, null), + ) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_remote_exec" "vault_write_license" { + for_each = toset([ + for idx, leader in local.leader : leader + if var.initialize_cluster + ]) + + depends_on = [ + enos_vault_unseal.leader, + enos_vault_unseal.maybe_force_unseal, + ] + + content = templatefile("${path.module}/templates/vault-write-license.sh", { + bin_path = local.bin_path, + root_token = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none") + license = coalesce(var.license, "none") + }) + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} diff --git a/enos/modules/vault_cluster/outputs.tf b/enos/modules/vault_cluster/outputs.tf new file mode 100644 index 000000000000..8e72ef44668b --- /dev/null +++ b/enos/modules/vault_cluster/outputs.tf @@ -0,0 +1,55 @@ +output "public_ips" { + description = "Vault cluster target host public_ips" + value = [for host in var.target_hosts : host.public_ip] +} + +output "private_ips" { + description = "Vault cluster target host private_ips" + value = [for host in var.target_hosts : host.private_ip] +} + +output "target_hosts" { + description = "The vault cluster instances that were created" + + value = var.target_hosts +} +output "root_token" { + value = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none") +} + +output "unseal_keys_b64" { + value = try(enos_vault_init.leader[0].unseal_keys_b64, []) +} + +output "unseal_keys_hex" { + value = try(enos_vault_init.leader[0].unseal_keys_hex, null) +} + +output "unseal_shares" { + value = try(enos_vault_init.leader[0].unseal_keys_shares, -1) +} + +output "unseal_threshold" { + value = try(enos_vault_init.leader[0].unseal_keys_threshold, -1) +} + +output "recovery_keys_b64" { + value = try(enos_vault_init.leader[0].recovery_keys_b64, []) +} + +output "recovery_keys_hex" { + value = try(enos_vault_init.leader[0].recovery_keys_hex, []) +} + +output "recovery_key_shares" { + value = try(enos_vault_init.leader[0].recovery_keys_shares, -1) +} + +output "recovery_threshold" { + value = try(enos_vault_init.leader[0].recovery_keys_threshold, -1) +} + +output "cluster_name" { + description = "The Vault cluster name" + value = var.cluster_name +} diff --git a/enos/modules/vault_cluster/templates/install-packages.sh b/enos/modules/vault_cluster/templates/install-packages.sh new file mode 100755 index 000000000000..61b6a1272d09 --- /dev/null +++ b/enos/modules/vault_cluster/templates/install-packages.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -ex -o pipefail + +packages="${packages}" + +if [ "$packages" == "" ] +then + echo "No dependencies to install." + exit 0 +fi + +function retry { + local retries=$1 + shift + local count=0 + + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + return "$exit" + fi + done + + return 0 +} + +echo "Installing Dependencies: $packages" +if [ -f /etc/debian_version ]; then + # Make sure cloud-init is not modifying our sources list while we're trying + # to install. + retry 7 grep ec2 /etc/apt/sources.list + + cd /tmp + retry 5 sudo apt update + retry 5 sudo apt install -y "$${packages[@]}" +else + cd /tmp + retry 7 sudo yum -y install "$${packages[@]}" +fi diff --git a/enos/modules/vault_cluster/templates/vault-write-license.sh b/enos/modules/vault_cluster/templates/vault-write-license.sh new file mode 100755 index 000000000000..10de84ba56f1 --- /dev/null +++ b/enos/modules/vault_cluster/templates/vault-write-license.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +license='${license}' +if test $license = "none"; then + exit 0 +fi + +function retry { + local retries=$1 + shift + local count=0 + + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + return "$exit" + fi + done + + return 0 +} + +export VAULT_ADDR=http://localhost:8200 +export VAULT_TOKEN='${root_token}' + +# Temporary hack until we can make the unseal resource handle legacy license +# setting. If we're running 1.8 and above then we shouldn't try to set a license. +ver=$(${bin_path} version) +if [[ "$(echo "$ver" |awk '{print $2}' |awk -F'.' '{print $2}')" -ge 8 ]]; then + exit 0 +fi + +retry 5 ${bin_path} write /sys/license text="$license" diff --git a/enos/modules/vault_cluster/variables.tf b/enos/modules/vault_cluster/variables.tf new file mode 100644 index 000000000000..d7be243e2efb --- /dev/null +++ b/enos/modules/vault_cluster/variables.tf @@ -0,0 +1,176 @@ +variable "artifactory_release" { + type = object({ + username = string + token = string + url = string + sha256 = string + }) + description = "The Artifactory release information to install Vault artifacts from Artifactory" + default = null +} + +variable "awskms_unseal_key_arn" { + type = string + description = "The AWSKMS key ARN if using the awskms unseal method" + default = null +} + +variable "cluster_name" { + type = string + description = "The Vault cluster name" + default = null +} + +variable "config_dir" { + type = string + description = "The directory to use for Vault configuration" + default = "/etc/vault.d" +} + +variable "config_env_vars" { + description = "Optional Vault configuration environment variables to set starting Vault" + type = map(string) + default = null +} + +variable "consul_cluster_tag" { + type = string + description = "The retry_join tag to use for Consul" + default = null +} + +variable "consul_data_dir" { + type = string + description = "The directory where the consul will store data" + default = "/opt/consul/data" +} + +variable "consul_install_dir" { + type = string + description = "The directory where the consul binary will be installed" + default = "/opt/consul/bin" +} + +variable "consul_log_file" { + type = string + description = "The file where the consul will write log output" + default = "/var/log/consul.log" +} + +variable "consul_release" { + type = object({ + version = string + edition = string + }) + description = "Consul release version and edition to install from releases.hashicorp.com" + default = { + version = "1.15.1" + edition = "oss" + } +} + +variable "force_unseal" { + type = bool + description = "Always unseal the Vault cluster even if we're not initializing it" + default = false +} + +variable "initialize_cluster" { + type = bool + description = "Initialize the Vault cluster" + default = true +} + +variable "install_dir" { + type = string + description = "The directory where the vault binary will be installed" + default = "/opt/vault/bin" +} + +variable "license" { + type = string + sensitive = true + description = "The value of the Vault license" + default = null +} + +variable "local_artifact_path" { + type = string + description = "The path to a locally built vault artifact to install. It can be a zip archive, RPM, or Debian package" + default = null +} + +variable "manage_service" { + type = bool + description = "Manage the Vault service users and systemd unit. Disable this to use configuration in RPM and Debian packages" + default = true +} + +variable "packages" { + type = list(string) + description = "A list of packages to install via the target host package manager" + default = [] +} + +variable "release" { + type = object({ + version = string + edition = string + }) + description = "Vault release version and edition to install from releases.hashicorp.com" + default = null +} + +variable "root_token" { + type = string + description = "The Vault root token that we can use to intialize and configure the cluster" + default = null +} + +variable "shamir_unseal_keys" { + type = list(string) + description = "Shamir unseal keys. Often only used adding additional nodes to an already initialized cluster." + default = null +} + +variable "storage_backend" { + type = string + description = "The storage backend to use" + default = "raft" + + validation { + condition = contains(["raft", "consul"], var.storage_backend) + error_message = "The storage_backend must be either raft or consul. No other storage backends are supported." + } +} + +variable "storage_backend_addl_config" { + type = map(any) + description = "An optional set of key value pairs to inject into the storage block" + default = {} +} + +variable "storage_node_prefix" { + type = string + description = "A prefix to use for each node in the Vault storage configuration" + default = "node" +} + +variable "target_hosts" { + description = "The target machines host addresses to use for the Vault cluster" + type = map(object({ + private_ip = string + public_ip = string + })) +} + +variable "unseal_method" { + type = string + description = "The method by which to unseal the Vault cluster" + default = "awskms" + + validation { + condition = contains(["awskms", "shamir"], var.unseal_method) + error_message = "The unseal_method must be either awskms or shamir. No other unseal methods are supported." + } +}