From 23f5b16a6733b4477def419e38b2063118abe934 Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Tue, 28 Mar 2023 10:56:16 -0600 Subject: [PATCH 1/2] [QT-525] enos: use spot instances for Vault targets The previous strategy for provisioning infrastructure targets was to use the cheapest instances that could reliably perform as Vault cluster nodes. With this change we introduce a new model of bidding for spot fleet instances with the goal of costs savings and often more powerful instances. The spot fleet instance bidding has only been implemented for Vault clusters. Updating our Consul backend bidding will be handled in another PR. * Create a new `vault_cluster` module that handles installation, configuration, initializing, and unsealing Vault clusters. * Create a `target_ec2_instances` module that can provision a group of instances on-demand. * Create a `target_ec2_spot_fleet` module that can bid on a fleet of spot instances. * Extend every Enos scenario to utilize the spot fleet target acquisition strategy and the `vault_cluster` module. * Update our Enos CI modules to handle both the `aws-nuke` permissions and also the privileges to provision spot fleets. Signed-off-by: Ryan Cragun --- .gitignore | 6 +- enos/ci/service-user-iam/main.tf | 71 +++- enos/ci/service-user-iam/service-quotas.tf | 39 +- enos/enos-modules.hcl | 32 +- enos/enos-scenario-agent.hcl | 148 ++++--- enos/enos-scenario-autopilot.hcl | 248 ++++++----- enos/enos-scenario-replication.hcl | 354 +++++++++------- enos/enos-scenario-smoke.hcl | 140 ++++--- enos/enos-scenario-ui.hcl | 117 ++++-- enos/enos-scenario-upgrade.hcl | 139 ++++--- enos/modules/target_ec2_instances/main.tf | 171 ++++++++ enos/modules/target_ec2_instances/outputs.tf | 11 + .../modules/target_ec2_instances/variables.tf | 61 +++ enos/modules/target_ec2_spot_fleet/main.tf | 388 ++++++++++++++++++ enos/modules/target_ec2_spot_fleet/outputs.tf | 11 + .../target_ec2_spot_fleet/variables.tf | 88 ++++ enos/modules/vault_cluster/main.tf | 335 +++++++++++++++ enos/modules/vault_cluster/outputs.tf | 55 +++ .../templates/install-packages.sh | 44 ++ .../templates/vault-write-license.sh | 38 ++ enos/modules/vault_cluster/variables.tf | 176 ++++++++ 21 files changed, 2177 insertions(+), 495 deletions(-) create mode 100644 enos/modules/target_ec2_instances/main.tf create mode 100644 enos/modules/target_ec2_instances/outputs.tf create mode 100644 enos/modules/target_ec2_instances/variables.tf create mode 100644 enos/modules/target_ec2_spot_fleet/main.tf create mode 100644 enos/modules/target_ec2_spot_fleet/outputs.tf create mode 100644 enos/modules/target_ec2_spot_fleet/variables.tf create mode 100644 enos/modules/vault_cluster/main.tf create mode 100644 enos/modules/vault_cluster/outputs.tf create mode 100755 enos/modules/vault_cluster/templates/install-packages.sh create mode 100755 enos/modules/vault_cluster/templates/vault-write-license.sh create mode 100644 enos/modules/vault_cluster/variables.tf diff --git a/.gitignore b/.gitignore index cdd542d60689..6138dcc5dd2a 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,10 @@ enos/.terraform/* enos/.terraform.lock.hcl enos/*.tfstate enos/*.tfstate.* +enos/**/.terraform/* +enos/**/.terraform.lock.hcl +enos/**/*.tfstate +enos/**/*.tfstate.* .DS_Store .idea @@ -127,4 +131,4 @@ website/components/node_modules .releaser/ *.log -tools/godoctests/.bin \ No newline at end of file +tools/godoctests/.bin diff --git a/enos/ci/service-user-iam/main.tf b/enos/ci/service-user-iam/main.tf index 6aafd9a3819f..c70dbaa13be5 100644 --- a/enos/ci/service-user-iam/main.tf +++ b/enos/ci/service-user-iam/main.tf @@ -31,6 +31,7 @@ resource "aws_iam_role" "role" { data "aws_iam_policy_document" "assume_role_policy_document" { provider = aws.us_east_1 + statement { effect = "Allow" actions = ["sts:AssumeRole"] @@ -46,11 +47,47 @@ resource "aws_iam_role_policy" "role_policy" { provider = aws.us_east_1 role = aws_iam_role.role.name name = "${local.service_user}_policy" - policy = data.aws_iam_policy_document.iam_policy_document.json + policy = data.aws_iam_policy_document.role_policy.json +} + +data "aws_iam_policy_document" "role_policy" { + source_policy_documents = [ + data.aws_iam_policy_document.enos_scenario.json, + data.aws_iam_policy_document.aws_nuke.json, + ] +} + +data "aws_iam_policy_document" "aws_nuke" { + provider = aws.us_east_1 + + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInternetGateways", + "ec2:DescribeNatGateways", + "ec2:DescribeRegions", + "ec2:DescribeVpnGateways", + "iam:DeleteAccessKey", + "iam:DeleteUser", + "iam:DeleteUserPolicy", + "iam:GetUser", + "iam:ListAccessKeys", + "iam:ListAccountAliases", + "iam:ListGroupsForUser", + "iam:ListUserPolicies", + "iam:ListUserTags", + "iam:ListUsers", + "iam:UntagUser", + "servicequotas:ListServiceQuotas" + ] + + resources = ["*"] + } } -data "aws_iam_policy_document" "iam_policy_document" { +data "aws_iam_policy_document" "enos_scenario" { provider = aws.us_east_1 + statement { effect = "Allow" actions = [ @@ -58,19 +95,27 @@ data "aws_iam_policy_document" "iam_policy_document" { "ec2:AttachInternetGateway", "ec2:AuthorizeSecurityGroupEgress", "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotFleetRequests", + "ec2:CancelSpotInstanceRequests", "ec2:CreateInternetGateway", "ec2:CreateKeyPair", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", "ec2:CreateRoute", "ec2:CreateRouteTable", "ec2:CreateSecurityGroup", + "ec2:CreateSpotDatafeedSubscription", "ec2:CreateSubnet", "ec2:CreateTags", "ec2:CreateVolume", "ec2:CreateVPC", "ec2:DeleteInternetGateway", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", "ec2:DeleteKeyPair", "ec2:DeleteRouteTable", "ec2:DeleteSecurityGroup", + "ec2:DeleteSpotDatafeedSubscription", "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DeleteVolume", @@ -84,14 +129,22 @@ data "aws_iam_policy_document" "iam_policy_document" { "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeInstanceTypes", "ec2:DescribeInternetGateways", - "ec2:DescribeInternetGateways", "ec2:DescribeKeyPairs", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", "ec2:DescribeNatGateways", "ec2:DescribeNetworkAcls", "ec2:DescribeNetworkInterfaces", "ec2:DescribeRegions", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotDatafeedSubscription", + "ec2:DescribeSpotFleetInstances", + "ec2:DescribeSpotFleetInstanceRequests", + "ec2:DescribeSpotFleetRequests", + "ec2:DescribeSpotFleetRequestHistory", + "ec2:DescribeSpotInstanceRequests", + "ec2:DescribeSpotPriceHistory", "ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVolumes", @@ -102,14 +155,21 @@ data "aws_iam_policy_document" "iam_policy_document" { "ec2:DescribeVpnGateways", "ec2:DetachInternetGateway", "ec2:DisassociateRouteTable", + "ec2:GetLaunchTemplateData", + "ec2:GetSpotPlacementScores", "ec2:ImportKeyPair", "ec2:ModifyInstanceAttribute", + "ec2:ModifyLaunchTemplate", + "ec2:ModifySpotFleetRequest", "ec2:ModifySubnetAttribute", "ec2:ModifyVPCAttribute", + "ec2:RequestSpotInstances", + "ec2:RequestSpotFleet", "ec2:ResetInstanceAttribute", "ec2:RevokeSecurityGroupEgress", "ec2:RevokeSecurityGroupIngress", "ec2:RunInstances", + "ec2:SendSpotInstanceInterruptions", "ec2:TerminateInstances", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeTargetGroups", @@ -118,11 +178,10 @@ data "aws_iam_policy_document" "iam_policy_document" { "iam:CreateInstanceProfile", "iam:CreatePolicy", "iam:CreateRole", - "iam:CreateRole", + "iam:CreateServiceLinkedRole", "iam:DeleteInstanceProfile", "iam:DeletePolicy", "iam:DeleteRole", - "iam:DeleteRole", "iam:DeleteRolePolicy", "iam:DetachRolePolicy", "iam:GetInstanceProfile", @@ -135,7 +194,6 @@ data "aws_iam_policy_document" "iam_policy_document" { "iam:ListPolicies", "iam:ListRolePolicies", "iam:ListRoles", - "iam:ListRoles", "iam:PassRole", "iam:PutRolePolicy", "iam:RemoveRoleFromInstanceProfile", @@ -153,6 +211,7 @@ data "aws_iam_policy_document" "iam_policy_document" { "kms:ScheduleKeyDeletion", "servicequotas:ListServiceQuotas" ] + resources = ["*"] } } diff --git a/enos/ci/service-user-iam/service-quotas.tf b/enos/ci/service-user-iam/service-quotas.tf index 57ab9aa415b2..656e38e2c18a 100644 --- a/enos/ci/service-user-iam/service-quotas.tf +++ b/enos/ci/service-user-iam/service-quotas.tf @@ -4,33 +4,62 @@ locals { // This is the code of the service quota to request a change for. Each adjustable limit has a // unique code. See, https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicequotas_service_quota#quota_code - subnets_per_vps_quota = "L-F678F1CE" + subnets_per_vpcs_quota = "L-F678F1CE" + standard_spot_instance_requests_quota = "L-34B43A08" } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_1" { provider = aws.us_east_2 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_2" { provider = aws.us_east_2 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_1" { provider = aws.us_west_1 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_2" { provider = aws.us_west_2 - quota_code = local.subnets_per_vps_quota + quota_code = local.subnets_per_vpcs_quota service_code = "vpc" value = 50 } + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_1" { + provider = aws.us_east_2 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_2" { + provider = aws.us_east_2 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_1" { + provider = aws.us_west_1 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} + +resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_2" { + provider = aws.us_west_2 + quota_code = local.standard_spot_instance_requests_quota + service_code = "ec2" + value = 640 +} diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index bfb69a2bdac0..465acaf271df 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -68,6 +68,27 @@ module "shutdown_multiple_nodes" { source = "./modules/shutdown_multiple_nodes" } +module "target_ec2_instances" { + source = "./modules/target_ec2_instances" + + common_tags = var.tags + instance_count = var.vault_instance_count + project_name = var.project_name + ssh_keypair = var.aws_ssh_keypair_name +} + +module "target_ec2_spot_fleet" { + source = "./modules/target_ec2_spot_fleet" + + common_tags = var.tags + instance_mem_min = 4096 + instance_cpu_min = 2 + project_name = var.project_name + // Current on-demand cost of t3.medium in us-east. + spot_price_max = "0.0416" + ssh_keypair = var.aws_ssh_keypair_name +} + module "vault_agent" { source = "./modules/vault_agent" @@ -75,7 +96,6 @@ module "vault_agent" { vault_instance_count = var.vault_instance_count } - module "vault_verify_agent_output" { source = "./modules/vault_verify_agent_output" @@ -83,15 +103,9 @@ module "vault_verify_agent_output" { } module "vault_cluster" { - source = "app.terraform.io/hashicorp-qti/aws-vault/enos" - # source = "../../terraform-enos-aws-vault" + source = "./modules/vault_cluster" - common_tags = var.tags - environment = "ci" - instance_count = var.vault_instance_count - project_name = var.project_name - ssh_aws_keypair = var.aws_ssh_keypair_name - vault_install_dir = var.vault_install_dir + install_dir = var.vault_install_dir } module "vault_get_cluster_ips" { diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index 6049fec23573..7b643ec9a951 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -25,13 +25,18 @@ scenario "agent" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } install_artifactory_artifact = local.bundle_path == null + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -54,21 +59,21 @@ scenario "agent" { module = "build_${matrix.artifact_source}" variables { - build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition]) - bundle_path = local.bundle_path - goarch = matrix.arch - goos = "linux" - artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null - artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null - artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null - artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null - arch = matrix.artifact_source == "artifactory" ? matrix.arch : null - vault_product_version = var.vault_product_version - artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null - distro = matrix.artifact_source == "artifactory" ? matrix.distro : null - edition = matrix.artifact_source == "artifactory" ? matrix.edition : null - instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null - revision = var.vault_revision + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition] + bundle_path = local.bundle_path + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null + artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null + artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.artifact_source == "artifactory" ? matrix.arch : null + product_version = var.vault_product_version + artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null + distro = matrix.artifact_source == "artifactory" ? matrix.distro : null + edition = matrix.artifact_source == "artifactory" ? matrix.edition : null + instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null + revision = var.vault_revision } } @@ -102,28 +107,29 @@ scenario "agent" { } } - step "create_backend_cluster" { - module = "backend_raft" + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances depends_on = [step.create_vpc] providers = { - enos = provider.enos.ubuntu + enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids["ubuntu"]["amd64"] - common_tags = local.tags - instance_type = var.backend_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - vpc_id = step.create_vpc.vpc_id + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id } } step "create_vault_cluster" { module = module.vault_cluster depends_on = [ - step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -131,28 +137,25 @@ scenario "agent" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags - consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = "raft" - unseal_method = "shamir" - vault_local_artifact_path = local.bundle_path - vault_artifactory_release = local.install_artifactory_artifact ? step.build_vault.vault_artifactory_release : null - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { VAULT_LOG_LEVEL = var.vault_log_level } + install_dir = var.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = "raft" + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = "shamir" } } step "start_vault_agent" { module = "vault_agent" depends_on = [ - step.create_backend_cluster, step.build_vault, step.create_vault_cluster, ] @@ -162,8 +165,8 @@ scenario "agent" { } variables { - vault_instances = step.create_vault_cluster.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token vault_agent_template_destination = "/tmp/agent_output.txt" vault_agent_template_contents = "{{ with secret \\\"auth/token/lookup-self\\\" }}orphan={{ .Data.orphan }} display_name={{ .Data.display_name }}{{ end }}" } @@ -181,49 +184,64 @@ scenario "agent" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_agent_template_destination = "/tmp/agent_output.txt" vault_agent_expected_output = "orphan=true display_name=approle" } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awkms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name } - output "vault_cluster_priv_ips" { + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts + } + + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token } - output "vault_cluster_unseal_keys_b64" { - description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_unseal_keys_hex" { - description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 + } + + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex } } diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index 1f46c92c9502..69d6e793e1aa 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -15,6 +15,12 @@ scenario "autopilot" { edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] artifact_type = ["package"] } + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } } terraform_cli = terraform_cli.default @@ -32,12 +38,17 @@ scenario "autopilot" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -108,36 +119,52 @@ scenario "autopilot" { } } - # This step creates a Vault cluster using a bundle downloaded from - # releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ - step.create_vpc, step.build_vault, + step.create_vault_cluster_targets ] + providers = { enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = "raft" + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + packages = local.packages + release = var.vault_autopilot_initial_release + storage_backend = "raft" storage_backend_addl_config = { autopilot_upgrade_version = var.vault_autopilot_initial_release.version } - unseal_method = matrix.seal - vault_install_dir = local.vault_install_dir - vault_release = var.vault_autopilot_initial_release - vault_license = step.read_license.license - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = matrix.seal } } @@ -155,12 +182,13 @@ scenario "autopilot" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } + step "verify_write_test_data" { module = module.vault_verify_write_data depends_on = [ @@ -175,9 +203,9 @@ scenario "autopilot" { variables { leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -189,8 +217,25 @@ scenario "autopilot" { } } - # This step creates a new Vault cluster using a bundle or package - # from the matrix.artifact_source, with the var.vault_product_version + step "create_vault_cluster_upgrade_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + cluster_name = step.create_vault_cluster_targets.cluster_name + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "upgrade_vault_cluster_with_autopilot" { module = module.vault_cluster depends_on = [ @@ -205,28 +250,25 @@ scenario "autopilot" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } + force_unseal = matrix.seal == "shamir" + initialize_cluster = false + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + root_token = step.create_vault_cluster.root_token + shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null storage_backend = "raft" storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config + storage_node_prefix = "upgrade_node" + target_hosts = step.create_vault_cluster_upgrade_targets.hosts unseal_method = matrix.seal - vault_cluster_tag = step.create_vault_cluster.vault_cluster_tag - vault_init = false - vault_install_dir = local.vault_install_dir - vault_license = step.read_license.license - vault_local_artifact_path = local.bundle_path - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_node_prefix = "upgrade_node" - vault_root_token = step.create_vault_cluster.vault_root_token - vault_unseal_when_no_init = matrix.seal == "shamir" - vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } } } @@ -234,6 +276,7 @@ scenario "autopilot" { module = module.vault_verify_unsealed depends_on = [ step.create_vault_cluster, + step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, ] @@ -243,7 +286,7 @@ scenario "autopilot" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances + vault_instances = step.create_vault_cluster_upgrade_targets.hosts } } @@ -260,14 +303,15 @@ scenario "autopilot" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.upgrade_vault_cluster_with_autopilot.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token } } step "verify_autopilot_await_server_removal_state" { module = module.vault_verify_autopilot depends_on = [ + step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, step.verify_raft_auto_join_voter ] @@ -280,8 +324,8 @@ scenario "autopilot" { vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_autopilot_upgrade_status = "await-server-removal" vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token } } @@ -289,6 +333,7 @@ scenario "autopilot" { module = module.vault_get_cluster_ips depends_on = [ step.create_vault_cluster, + step.create_vault_cluster_upgrade_targets, step.get_vault_cluster_ips, step.upgrade_vault_cluster_with_autopilot ] @@ -298,11 +343,11 @@ scenario "autopilot" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - added_vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token node_public_ip = step.get_vault_cluster_ips.leader_public_ip + added_vault_instances = step.create_vault_cluster_targets.hosts } } @@ -329,6 +374,7 @@ scenario "autopilot" { step "raft_remove_peers" { module = module.vault_raft_remove_peer depends_on = [ + step.create_vault_cluster_upgrade_targets, step.get_updated_vault_cluster_ips, step.upgrade_vault_cluster_with_autopilot, step.verify_autopilot_await_server_removal_state @@ -339,11 +385,11 @@ scenario "autopilot" { } variables { - vault_install_dir = local.vault_install_dir operator_instance = step.get_updated_vault_cluster_ips.leader_public_ip - remove_vault_instances = step.create_vault_cluster.vault_instances + remove_vault_instances = step.create_vault_cluster_targets.hosts + vault_install_dir = local.vault_install_dir vault_instance_count = 3 - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -359,7 +405,7 @@ scenario "autopilot" { } variables { - old_vault_instances = step.create_vault_cluster.vault_instances + old_vault_instances = step.create_vault_cluster_targets.hosts vault_instance_count = 3 } } @@ -367,6 +413,7 @@ scenario "autopilot" { step "verify_autopilot_idle_state" { module = module.vault_verify_autopilot depends_on = [ + step.create_vault_cluster_upgrade_targets, step.upgrade_vault_cluster_with_autopilot, step.verify_raft_auto_join_voter, step.remove_old_nodes @@ -380,15 +427,16 @@ scenario "autopilot" { vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_autopilot_upgrade_status = "idle" vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } step "verify_undo_logs_status" { - skip_step = try(semverconstraint(var.vault_product_version, "<1.13.0-0"), true) + skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0") module = module.vault_verify_undo_logs depends_on = [ + step.create_vault_cluster_upgrade_targets, step.remove_old_nodes, step.upgrade_vault_cluster_with_autopilot, step.verify_autopilot_idle_state @@ -400,78 +448,78 @@ scenario "autopilot" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_upgrade_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name } - output "vault_cluster_priv_ips" { - description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "private_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.private_ips } - output "vault_cluster_root_token" { - description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_unseal_keys_b64" { - description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + output "root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.root_token } - output "vault_cluster_recovery_key_shares" { + output "recovery_key_shares" { description = "The Vault cluster recovery key shares" - value = step.create_vault_cluster.vault_recovery_key_shares + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_recovery_keys_b64" { + output "recovery_keys_b64" { description = "The Vault cluster recovery keys b64" - value = step.create_vault_cluster.vault_recovery_keys_b64 + value = step.create_vault_cluster.recovery_keys_b64 } - output "vault_cluster_recovery_keys_hex" { + output "recovery_keys_hex" { description = "The Vault cluster recovery keys hex" - value = step.create_vault_cluster.vault_recovery_keys_hex + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_hex" { - description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex } - output "upgraded_vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.upgrade_vault_cluster_with_autopilot.instance_ids + output "upgrade_hosts" { + description = "The Vault cluster target hosts" + value = step.upgrade_vault_cluster_with_autopilot.target_hosts } - output "upgraded_vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.upgrade_vault_cluster_with_autopilot.instance_public_ips + output "upgrade_private_ips" { + description = "The Vault cluster private IPs" + value = step.upgrade_vault_cluster_with_autopilot.private_ips } - output "upgraded_vault_cluster_priv_ips" { - description = "The Vault cluster private IPs" - value = step.upgrade_vault_cluster_with_autopilot.instance_private_ips + output "upgrade_public_ips" { + description = "The Vault cluster public IPs" + value = step.upgrade_vault_cluster_with_autopilot.public_ips } } diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl index 610ecf3cf7db..79a586c1e574 100644 --- a/enos/enos-scenario-replication.hcl +++ b/enos/enos-scenario-replication.hcl @@ -22,6 +22,12 @@ scenario "replication" { edition = ["ent.fips1402", "ent.hsm.fips1402"] artifact_type = ["package"] } + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } } terraform_cli = terraform_cli.default @@ -39,12 +45,17 @@ scenario "replication" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -134,37 +145,55 @@ scenario "replication" { } } - step "create_vault_primary_cluster" { + step "create_primary_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_primary_cluster" { module = module.vault_cluster depends_on = [ step.create_primary_backend_cluster, step.build_vault, + step.create_primary_cluster_targets ] + providers = { enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_primary_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag consul_release = matrix.primary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.primary_backend - unseal_method = matrix.primary_seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_license = step.read_license.license - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = matrix.primary_backend + target_hosts = step.create_primary_cluster_targets.hosts + unseal_method = matrix.primary_seal } } @@ -189,44 +218,62 @@ scenario "replication" { } } - step "create_vault_secondary_cluster" { + step "create_secondary_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + + step "create_secondary_cluster" { module = module.vault_cluster depends_on = [ step.create_secondary_backend_cluster, step.build_vault, + step.create_secondary_cluster_targets ] + providers = { enos = local.enos_provider[matrix.distro] } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_secondary_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_secondary_backend_cluster.consul_cluster_tag consul_release = matrix.secondary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.secondary_backend - unseal_method = matrix.secondary_seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_license = step.read_license.license - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = matrix.secondary_backend + target_hosts = step.create_secondary_cluster_targets.hosts + unseal_method = matrix.secondary_seal } } - step "verify_vault_primary_unsealed" { + step "verify_that_vault_primary_cluster_is_unsealed" { module = module.vault_verify_unsealed depends_on = [ - step.create_vault_primary_cluster + step.create_primary_cluster ] providers = { @@ -234,15 +281,15 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } - step "verify_vault_secondary_unsealed" { + step "verify_that_vault_secondary_cluster_is_unsealed" { module = module.vault_verify_unsealed depends_on = [ - step.create_vault_secondary_cluster + step.create_secondary_cluster ] providers = { @@ -250,42 +297,42 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_secondary_cluster.vault_instances + vault_instances = step.create_secondary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } step "get_primary_cluster_ips" { module = module.vault_get_cluster_ips - depends_on = [step.verify_vault_primary_unsealed] + depends_on = [step.verify_that_vault_primary_cluster_is_unsealed] providers = { enos = local.enos_provider[matrix.distro] } variables { - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } step "get_secondary_cluster_ips" { module = module.vault_get_cluster_ips - depends_on = [step.verify_vault_secondary_unsealed] + depends_on = [step.verify_that_vault_secondary_cluster_is_unsealed] providers = { enos = local.enos_provider[matrix.distro] } variables { - vault_instances = step.create_vault_secondary_cluster.vault_instances + vault_instances = step.create_secondary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_secondary_cluster.vault_root_token + vault_root_token = step.create_secondary_cluster.root_token } } - step "verify_vault_primary_write_data" { + step "write_test_data_on_primary" { module = module.vault_verify_write_data depends_on = [step.get_primary_cluster_ips] @@ -297,9 +344,9 @@ scenario "replication" { variables { leader_public_ip = step.get_primary_cluster_ips.leader_public_ip leader_private_ip = step.get_primary_cluster_ips.leader_private_ip - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -307,7 +354,7 @@ scenario "replication" { module = module.vault_setup_perf_primary depends_on = [ step.get_primary_cluster_ips, - step.verify_vault_primary_write_data + step.write_test_data_on_primary ] providers = { @@ -318,7 +365,7 @@ scenario "replication" { primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip primary_leader_private_ip = step.get_primary_cluster_ips.leader_private_ip vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -333,7 +380,7 @@ scenario "replication" { variables { primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -349,7 +396,7 @@ scenario "replication" { secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip secondary_leader_private_ip = step.get_secondary_cluster_ips.leader_private_ip vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_secondary_cluster.vault_root_token + vault_root_token = step.create_secondary_cluster.root_token wrapping_token = step.generate_secondary_token.secondary_token } } @@ -359,8 +406,8 @@ scenario "replication" { step "unseal_secondary_followers" { module = module.vault_unseal_nodes depends_on = [ - step.create_vault_primary_cluster, - step.create_vault_secondary_cluster, + step.create_primary_cluster, + step.create_secondary_cluster, step.get_secondary_cluster_ips, step.configure_performance_replication_secondary ] @@ -372,12 +419,12 @@ scenario "replication" { variables { follower_public_ips = step.get_secondary_cluster_ips.follower_public_ips vault_install_dir = local.vault_install_dir - vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : step.create_vault_primary_cluster.vault_recovery_keys_hex + vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : step.create_primary_cluster.recovery_keys_hex vault_seal_type = matrix.primary_seal == "shamir" ? matrix.primary_seal : matrix.secondary_seal } } - step "verify_vault_secondary_unsealed_after_replication" { + step "verify_secondary_cluster_is_unsealed_after_enabling_replication" { module = module.vault_verify_unsealed depends_on = [ step.unseal_secondary_followers @@ -388,14 +435,14 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_secondary_cluster.vault_instances + vault_instances = step.create_secondary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } step "verify_performance_replication" { module = module.vault_verify_performance_replication - depends_on = [step.verify_vault_secondary_unsealed_after_replication] + depends_on = [step.verify_secondary_cluster_is_unsealed_after_enabling_replication] providers = { enos = local.enos_provider[matrix.distro] @@ -415,7 +462,7 @@ scenario "replication" { depends_on = [ step.verify_performance_replication, step.get_secondary_cluster_ips, - step.verify_vault_primary_write_data + step.write_test_data_on_primary ] providers = { @@ -428,13 +475,32 @@ scenario "replication" { } } - step "add_primary_cluster_nodes" { + step "create_more_primary_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + + step "add_more_nodes_to_primary_cluster" { module = module.vault_cluster depends_on = [ step.create_vpc, step.create_primary_backend_cluster, - step.create_vault_primary_cluster, - step.verify_replicated_data + step.create_primary_cluster, + step.verify_replicated_data, + step.create_more_primary_cluster_targets ] providers = { @@ -442,45 +508,42 @@ scenario "replication" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_primary_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag consul_release = matrix.primary_backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.primary_backend - unseal_method = matrix.primary_seal - vault_cluster_tag = step.create_vault_primary_cluster.vault_cluster_tag - vault_init = false - vault_license = step.read_license.license - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_node_prefix = "newprimary_node" - vault_root_token = step.create_vault_primary_cluster.vault_root_token - vault_unseal_when_no_init = matrix.primary_seal == "shamir" - vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } - } - } - - step "verify_add_node_unsealed" { + force_unseal = matrix.primary_seal == "shamir" + initialize_cluster = false + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + root_token = step.create_primary_cluster.root_token + shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null + storage_backend = matrix.primary_backend + storage_node_prefix = "newprimary_node" + target_hosts = step.create_more_primary_cluster_targets.hosts + unseal_method = matrix.primary_seal + } + } + + step "verify_more_primary_nodes_unsealed" { module = module.vault_verify_unsealed - depends_on = [step.add_primary_cluster_nodes] + depends_on = [step.add_more_nodes_to_primary_cluster] providers = { enos = local.enos_provider[matrix.distro] } variables { - vault_instances = step.add_primary_cluster_nodes.vault_instances + vault_instances = step.create_more_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } @@ -489,9 +552,9 @@ scenario "replication" { skip_step = matrix.primary_backend != "raft" module = module.vault_verify_raft_auto_join_voter depends_on = [ - step.add_primary_cluster_nodes, - step.create_vault_primary_cluster, - step.verify_add_node_unsealed + step.add_more_nodes_to_primary_cluster, + step.create_primary_cluster, + step.verify_more_primary_nodes_unsealed ] providers = { @@ -499,9 +562,9 @@ scenario "replication" { } variables { - vault_instances = step.add_primary_cluster_nodes.vault_instances + vault_instances = step.create_more_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_primary_cluster.vault_root_token + vault_root_token = step.create_primary_cluster.root_token } } @@ -509,7 +572,7 @@ scenario "replication" { module = module.shutdown_node depends_on = [ step.get_primary_cluster_ips, - step.verify_add_node_unsealed + step.verify_more_primary_nodes_unsealed ] providers = { @@ -540,7 +603,7 @@ scenario "replication" { step "get_updated_primary_cluster_ips" { module = module.vault_get_cluster_ips depends_on = [ - step.add_primary_cluster_nodes, + step.add_more_nodes_to_primary_cluster, step.remove_primary_follower_1, step.remove_primary_leader ] @@ -550,10 +613,10 @@ scenario "replication" { } variables { - vault_instances = step.create_vault_primary_cluster.vault_instances + vault_instances = step.create_primary_cluster_targets.hosts vault_install_dir = local.vault_install_dir - added_vault_instances = step.add_primary_cluster_nodes.vault_instances - vault_root_token = step.create_vault_primary_cluster.vault_root_token + added_vault_instances = step.create_more_primary_cluster_targets.hosts + vault_root_token = step.create_primary_cluster.root_token node_public_ip = step.get_primary_cluster_ips.follower_public_ip_2 } } @@ -575,112 +638,97 @@ scenario "replication" { } } - output "vault_primary_cluster_pub_ips" { - description = "The Vault primary cluster public IPs" - value = step.create_vault_primary_cluster.instance_public_ips + output "primary_cluster_hosts" { + description = "The Vault primary cluster target hosts" + value = step.create_primary_cluster_targets.hosts } - output "vault_primary_cluster_priv_ips" { - description = "The Vault primary cluster private IPs" - value = step.create_vault_primary_cluster.instance_private_ips + output "primary_cluster_additional_hosts" { + description = "The Vault added new node on primary cluster target hosts" + value = step.create_more_primary_cluster_targets.hosts } - output "vault_primary_newnode_pub_ip" { - description = "The Vault added new node on primary cluster public IP" - value = step.add_primary_cluster_nodes.instance_public_ips - } - - output "vault_primary_newnode_priv_ip" { - description = "The Vault added new node on primary cluster private IP" - value = step.add_primary_cluster_nodes.instance_private_ips - } - - output "vault_primary_cluster_root_token" { + output "primary_cluster_root_token" { description = "The Vault primary cluster root token" - value = step.create_vault_primary_cluster.vault_root_token + value = step.create_primary_cluster.root_token } - output "vault_primary_cluster_unseal_keys_b64" { + output "primary_cluster_unseal_keys_b64" { description = "The Vault primary cluster unseal keys" - value = step.create_vault_primary_cluster.vault_unseal_keys_b64 + value = step.create_primary_cluster.unseal_keys_b64 } - output "vault_primary_cluster_unseal_keys_hex" { + output "primary_cluster_unseal_keys_hex" { description = "The Vault primary cluster unseal keys hex" - value = step.create_vault_primary_cluster.vault_unseal_keys_hex + value = step.create_primary_cluster.unseal_keys_hex } - output "vault_primary_cluster_recovery_key_shares" { + output "primary_cluster_recovery_key_shares" { description = "The Vault primary cluster recovery key shares" - value = step.create_vault_primary_cluster.vault_recovery_key_shares + value = step.create_primary_cluster.recovery_key_shares } - output "vault_primary_cluster_recovery_keys_b64" { + output "primary_cluster_recovery_keys_b64" { description = "The Vault primary cluster recovery keys b64" - value = step.create_vault_primary_cluster.vault_recovery_keys_b64 + value = step.create_primary_cluster.recovery_keys_b64 } - output "vault_primary_cluster_recovery_keys_hex" { + output "primary_cluster_recovery_keys_hex" { description = "The Vault primary cluster recovery keys hex" - value = step.create_vault_primary_cluster.vault_recovery_keys_hex + value = step.create_primary_cluster.recovery_keys_hex } - output "vault_secondary_cluster_pub_ips" { + output "secondary_cluster_hosts" { description = "The Vault secondary cluster public IPs" - value = step.create_vault_secondary_cluster.instance_public_ips - } - - output "vault_secondary_cluster_priv_ips" { - description = "The Vault secondary cluster private IPs" - value = step.create_vault_secondary_cluster.instance_private_ips + value = step.create_secondary_cluster_targets.hosts } - output "vault_primary_performance_replication_status" { + output "initial_primary_replication_status" { description = "The Vault primary cluster performance replication status" value = step.verify_performance_replication.primary_replication_status } - output "vault_replication_known_primary_cluster_addrs" { + output "initial_known_primary_cluster_addresses" { description = "The Vault secondary cluster performance replication status" value = step.verify_performance_replication.known_primary_cluster_addrs } - output "vault_secondary_performance_replication_status" { + output "initial_secondary_performance_replication_status" { description = "The Vault secondary cluster performance replication status" value = step.verify_performance_replication.secondary_replication_status } - output "vault_primary_updated_performance_replication_status" { + output "intial_primary_replication_data_secondaries" { + description = "The Vault primary cluster secondaries connection status" + value = step.verify_performance_replication.primary_replication_data_secondaries + } + + output "initial_secondary_replication_data_primaries" { + description = "The Vault secondary cluster primaries connection status" + value = step.verify_performance_replication.secondary_replication_data_primaries + } + + output "updated_primary_replication_status" { description = "The Vault updated primary cluster performance replication status" value = step.verify_updated_performance_replication.primary_replication_status } - output "vault_updated_replication_known_primary_cluster_addrs" { + output "updated_known_primary_cluster_addresses" { description = "The Vault secondary cluster performance replication status" value = step.verify_updated_performance_replication.known_primary_cluster_addrs } - output "verify_secondary_updated_performance_replication_status" { + output "updated_secondary_replication_status" { description = "The Vault updated secondary cluster performance replication status" value = step.verify_updated_performance_replication.secondary_replication_status } - output "primary_replication_data_secondaries" { - description = "The Vault primary cluster secondaries connection status" - value = step.verify_performance_replication.primary_replication_data_secondaries - } - - output "secondary_replication_data_primaries" { - description = "The Vault secondary cluster primaries connection status" - value = step.verify_performance_replication.secondary_replication_data_primaries - } - - output "primary_updated_replication_data_secondaries" { + output "updated_primary_replication_data_secondaries" { description = "The Vault updated primary cluster secondaries connection status" value = step.verify_updated_performance_replication.primary_replication_data_secondaries } - output "secondary_updated_replication_data_primaries" { + output "updated_secondary_replication_data_primaries" { description = "The Vault updated secondary cluster primaries connection status" value = step.verify_updated_performance_replication.secondary_replication_data_primaries } diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index f2b5e9b5ef61..24283d4860ac 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -17,6 +17,12 @@ scenario "smoke" { edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"] artifact_type = ["package"] } + + # Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } } terraform_cli = terraform_cli.default @@ -35,12 +41,17 @@ scenario "smoke" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -137,11 +148,30 @@ scenario "smoke" { } } + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -149,26 +179,24 @@ scenario "smoke" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag consul_release = matrix.backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.backend - unseal_method = matrix.seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + packages = local.packages + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = matrix.seal } } @@ -181,9 +209,9 @@ scenario "smoke" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -196,13 +224,13 @@ scenario "smoke" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_edition = matrix.edition vault_install_dir = local.vault_install_dir vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -216,7 +244,7 @@ scenario "smoke" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts } } @@ -234,9 +262,9 @@ scenario "smoke" { variables { leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -251,8 +279,8 @@ scenario "smoke" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } @@ -267,7 +295,7 @@ scenario "smoke" { variables { vault_edition = matrix.edition vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts } } @@ -297,63 +325,63 @@ scenario "smoke" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts } - output "vault_cluster_priv_ips" { + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token } - output "vault_cluster_recovery_key_shares" { + output "recovery_key_shares" { description = "The Vault cluster recovery key shares" - value = step.create_vault_cluster.vault_recovery_key_shares + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_recovery_keys_b64" { + output "recovery_keys_b64" { description = "The Vault cluster recovery keys b64" - value = step.create_vault_cluster.vault_recovery_keys_b64 + value = step.create_vault_cluster.recovery_keys_b64 } - output "vault_cluster_recovery_keys_hex" { + output "recovery_keys_hex" { description = "The Vault cluster recovery keys hex" - value = step.create_vault_cluster.vault_recovery_keys_hex + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_b64" { + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_unseal_keys_hex" { + output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex - } - - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + value = step.create_vault_cluster.unseal_keys_hex } } diff --git a/enos/enos-scenario-ui.hcl b/enos/enos-scenario-ui.hcl index 6fba448e2d5b..3b54dcafd696 100644 --- a/enos/enos-scenario-ui.hcl +++ b/enos/enos-scenario-ui.hcl @@ -113,11 +113,29 @@ scenario "ui" { } } + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.create_vpc.ami_ids[local.distro][local.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -125,20 +143,22 @@ scenario "ui" { } variables { - ami_id = step.create_vpc.ami_ids[local.distro][local.arch] - common_tags = local.tags - consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.backend - unseal_method = local.seal - vault_local_artifact_path = local.bundle_path - vault_install_dir = local.vault_install_dir - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { VAULT_LOG_LEVEL = var.vault_log_level } + consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag + consul_release = matrix.backend == "consul" ? { + edition = var.backend_edition + version = local.consul_version + } : null + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + local_artifact_path = local.bundle_path + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = local.seal } } @@ -146,52 +166,72 @@ scenario "ui" { module = module.vault_test_ui variables { - vault_addr = step.create_vault_cluster.instance_public_ips[0] - vault_root_token = step.create_vault_cluster.vault_root_token - vault_unseal_keys = step.create_vault_cluster.vault_recovery_keys_b64 - vault_recovery_threshold = step.create_vault_cluster.vault_recovery_threshold + vault_addr = step.create_vault_cluster_targets.hosts[0].public_ip + vault_root_token = step.create_vault_cluster.root_token + vault_unseal_keys = step.create_vault_cluster.recovery_keys_b64 + vault_recovery_threshold = step.create_vault_cluster.recovery_threshold ui_test_filter = local.ui_test_filter } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts } - output "vault_cluster_priv_ips" { + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token + } + + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares + } + + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_b64" { + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_unseal_keys_hex" { + output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex + value = step.create_vault_cluster.unseal_keys_hex } - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + output "ui_test_environment" { + value = step.test_ui.ui_test_environment + description = "The environment variables that are required in order to run the test:enos yarn target" } output "ui_test_stderr" { @@ -203,9 +243,4 @@ scenario "ui" { description = "The stdout of the ui tests that ran" value = step.test_ui.ui_test_stdout } - - output "ui_test_environment" { - value = step.test_ui.ui_test_environment - description = "The environment variables that are required in order to run the test:enos yarn target" - } } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index 3b576ed70087..66f1e5436540 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -35,12 +35,17 @@ scenario "upgrade" { "ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"] "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"] } - bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null - dependencies_to_install = ["jq"] + bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null + packages = ["jq"] enos_provider = { rhel = provider.enos.rhel ubuntu = provider.enos.ubuntu } + spot_price_max = { + // These prices are based on on-demand cost for t3.medium in us-east + "rhel" = "0.1016" + "ubuntu" = "0.0416" + } tags = merge({ "Project Name" : var.project_name "Project" : "Enos", @@ -138,13 +143,30 @@ scenario "upgrade" { } } - # This step creates a Vault cluster using a bundle downloaded from - # releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release + step "create_vault_cluster_targets" { + module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + common_tags = local.tags + instance_type = local.vault_instance_type // only used for on-demand instances + spot_price_max = local.spot_price_max[matrix.distro] + vpc_id = step.create_vpc.vpc_id + } + } + step "create_vault_cluster" { module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, + step.create_vault_cluster_targets ] providers = { @@ -152,25 +174,24 @@ scenario "upgrade" { } variables { - ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch] - common_tags = local.tags + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + awskms_unseal_key_arn = step.create_vpc.kms_key_arn + cluster_name = step.create_vault_cluster_targets.cluster_name + config_env_vars = { + VAULT_LOG_LEVEL = var.vault_log_level + } consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag consul_release = matrix.backend == "consul" ? { edition = var.backend_edition version = matrix.consul_version } : null - dependencies_to_install = local.dependencies_to_install - instance_type = local.vault_instance_type - kms_key_arn = step.create_vpc.kms_key_arn - storage_backend = matrix.backend - unseal_method = matrix.seal - vault_install_dir = local.vault_install_dir - vault_release = var.vault_upgrade_initial_release - vault_license = matrix.edition != "oss" ? step.read_license.license : null - vpc_id = step.create_vpc.vpc_id - vault_environment = { - VAULT_LOG_LEVEL = var.vault_log_level - } + install_dir = local.vault_install_dir + license = matrix.edition != "oss" ? step.read_license.license : null + packages = local.packages + release = var.vault_upgrade_initial_release + storage_backend = matrix.backend + target_hosts = step.create_vault_cluster_targets.hosts + unseal_method = matrix.seal } } @@ -183,9 +204,9 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -203,9 +224,9 @@ scenario "upgrade" { variables { leader_public_ip = step.get_vault_cluster_ips.leader_public_ip leader_private_ip = step.get_vault_cluster_ips.leader_private_ip - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -223,11 +244,11 @@ scenario "upgrade" { variables { vault_api_addr = "http://localhost:8200" - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_local_artifact_path = local.bundle_path vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null vault_install_dir = local.vault_install_dir - vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null + vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null vault_seal_type = matrix.seal } } @@ -244,13 +265,13 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_edition = matrix.edition vault_install_dir = local.vault_install_dir vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -266,9 +287,9 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir - vault_root_token = step.create_vault_cluster.vault_root_token + vault_root_token = step.create_vault_cluster.root_token } } @@ -285,7 +306,7 @@ scenario "upgrade" { } variables { - vault_instances = step.create_vault_cluster.vault_instances + vault_instances = step.create_vault_cluster_targets.hosts vault_install_dir = local.vault_install_dir } } @@ -322,63 +343,63 @@ scenario "upgrade" { variables { vault_install_dir = local.vault_install_dir - vault_instances = step.create_vault_cluster.vault_instances - vault_root_token = step.create_vault_cluster.vault_root_token + vault_instances = step.create_vault_cluster_targets.hosts + vault_root_token = step.create_vault_cluster.root_token } } - output "vault_cluster_instance_ids" { - description = "The Vault cluster instance IDs" - value = step.create_vault_cluster.instance_ids + output "awskms_unseal_key_arn" { + description = "The Vault cluster KMS key arn" + value = step.create_vpc.kms_key_arn } - output "vault_cluster_pub_ips" { - description = "The Vault cluster public IPs" - value = step.create_vault_cluster.instance_public_ips + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name } - output "vault_cluster_priv_ips" { + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.target_hosts + } + + output "private_ips" { description = "The Vault cluster private IPs" - value = step.create_vault_cluster.instance_private_ips + value = step.create_vault_cluster.private_ips } - output "vault_cluster_key_id" { - description = "The Vault cluster Key ID" - value = step.create_vault_cluster.key_id + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips } - output "vault_cluster_root_token" { + output "root_token" { description = "The Vault cluster root token" - value = step.create_vault_cluster.vault_root_token + value = step.create_vault_cluster.root_token } - output "vault_cluster_recovery_key_shares" { + output "recovery_key_shares" { description = "The Vault cluster recovery key shares" - value = step.create_vault_cluster.vault_recovery_key_shares + value = step.create_vault_cluster.recovery_key_shares } - output "vault_cluster_recovery_keys_b64" { + output "recovery_keys_b64" { description = "The Vault cluster recovery keys b64" - value = step.create_vault_cluster.vault_recovery_keys_b64 + value = step.create_vault_cluster.recovery_keys_b64 } - output "vault_cluster_recovery_keys_hex" { + output "recovery_keys_hex" { description = "The Vault cluster recovery keys hex" - value = step.create_vault_cluster.vault_recovery_keys_hex + value = step.create_vault_cluster.recovery_keys_hex } - output "vault_cluster_unseal_keys_b64" { + output "unseal_keys_b64" { description = "The Vault cluster unseal keys" - value = step.create_vault_cluster.vault_unseal_keys_b64 + value = step.create_vault_cluster.unseal_keys_b64 } - output "vault_cluster_unseal_keys_hex" { + output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" - value = step.create_vault_cluster.vault_unseal_keys_hex - } - - output "vault_cluster_tag" { - description = "The Vault cluster tag" - value = step.create_vault_cluster.vault_cluster_tag + value = step.create_vault_cluster.unseal_keys_hex } } diff --git a/enos/modules/target_ec2_instances/main.tf b/enos/modules/target_ec2_instances/main.tf new file mode 100644 index 000000000000..06b44aad21f7 --- /dev/null +++ b/enos/modules/target_ec2_instances/main.tf @@ -0,0 +1,171 @@ +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.3.2" + } + } +} + +data "aws_vpc" "vpc" { + id = var.vpc_id +} + +data "aws_subnets" "vpc" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +data "aws_kms_key" "kms_key" { + key_id = var.awskms_unseal_key_arn +} + +data "aws_iam_policy_document" "target" { + statement { + resources = ["*"] + + actions = [ + "ec2:DescribeInstances", + "secretsmanager:*" + ] + } + + statement { + resources = [var.awskms_unseal_key_arn] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } +} + +data "aws_iam_policy_document" "target_instance_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "enos_environment" "localhost" {} + +resource "random_string" "cluster_name" { + length = 8 + lower = true + upper = false + numeric = false + special = false +} + +locals { + instances = toset([for idx in range(var.instance_count) : tostring(idx)]) + cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result) + name_prefix = "${var.project_name}-${local.cluster_name}" +} + +resource "aws_iam_role" "target_instance_role" { + name = "target_instance_role-${random_string.cluster_name.result}" + assume_role_policy = data.aws_iam_policy_document.target_instance_role.json +} + +resource "aws_iam_instance_profile" "target" { + name = "${local.name_prefix}-target" + role = aws_iam_role.target_instance_role.name +} + +resource "aws_iam_role_policy" "target" { + name = "${local.name_prefix}-target" + role = aws_iam_role.target_instance_role.id + policy = data.aws_iam_policy_document.target.json +} + +resource "aws_security_group" "target" { + name = "${local.name_prefix}-target" + description = "Target instance security group" + vpc_id = var.vpc_id + + # SSH traffic + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)] + } + + # Vault traffic + ingress { + from_port = 8200 + to_port = 8201 + protocol = "tcp" + cidr_blocks = flatten([ + "${data.enos_environment.localhost.public_ip_address}/32", + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + formatlist("%s/32", var.ssh_allow_ips)]) + } + + # Consul traffic + ingress { + from_port = 8301 + to_port = 8301 + protocol = "tcp" + cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)] + } + + ingress { + from_port = 8301 + to_port = 8301 + protocol = "udp" + cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)] + } + + # Internal traffic + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + } + + # External traffic + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = merge( + var.common_tags, + { + Name = "${local.name_prefix}-sg" + }, + ) +} + +resource "aws_instance" "targets" { + for_each = local.instances + ami = var.ami_id + instance_type = var.instance_type + vpc_security_group_ids = [aws_security_group.target.id] + subnet_id = tolist(data.aws_subnets.vpc.ids)[each.key % length(data.aws_subnets.vpc.ids)] + key_name = var.ssh_keypair + iam_instance_profile = aws_iam_instance_profile.target.name + tags = merge( + var.common_tags, + { + Name = "${local.name_prefix}-target-instance" + Type = local.cluster_name + }, + ) +} diff --git a/enos/modules/target_ec2_instances/outputs.tf b/enos/modules/target_ec2_instances/outputs.tf new file mode 100644 index 000000000000..9428bfdb9915 --- /dev/null +++ b/enos/modules/target_ec2_instances/outputs.tf @@ -0,0 +1,11 @@ +output "cluster_name" { + value = local.cluster_name +} + +output "hosts" { + description = "The ec2 instance target hosts" + value = { for idx in range(var.instance_count) : idx => { + public_ip = aws_instance.targets[idx].public_ip + private_ip = aws_instance.targets[idx].private_ip + } } +} diff --git a/enos/modules/target_ec2_instances/variables.tf b/enos/modules/target_ec2_instances/variables.tf new file mode 100644 index 000000000000..89dbbf03c776 --- /dev/null +++ b/enos/modules/target_ec2_instances/variables.tf @@ -0,0 +1,61 @@ +variable "ami_id" { + description = "The machine image identifier" + type = string +} + +variable "awskms_unseal_key_arn" { + type = string + description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" + default = null +} + +variable "cluster_name" { + type = string + description = "A unique cluster identifier" + default = null +} + +variable "common_tags" { + description = "Common tags for cloud resources" + type = map(string) + default = { "Project" : "Enos" } +} + +variable "instance_count" { + description = "The number of target instances to create" + type = number + default = 3 +} + +variable "instance_type" { + description = "The instance machine type" + type = string + default = "t3.small" +} + +variable "project_name" { + description = "A unique project name" + type = string +} + +variable "spot_price_max" { + description = "Unused shim variable to match target_ec2_spot_fleet" + type = string + default = null +} + +variable "ssh_allow_ips" { + description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" + type = list(string) + default = [] +} + +variable "ssh_keypair" { + description = "SSH keypair used to connect to EC2 instances" + type = string +} + +variable "vpc_id" { + description = "The identifier of the VPC where the target instances will be created" + type = string +} diff --git a/enos/modules/target_ec2_spot_fleet/main.tf b/enos/modules/target_ec2_spot_fleet/main.tf new file mode 100644 index 000000000000..56c786f5c6a1 --- /dev/null +++ b/enos/modules/target_ec2_spot_fleet/main.tf @@ -0,0 +1,388 @@ +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.3.2" + } + } +} + +data "aws_vpc" "vpc" { + id = var.vpc_id +} + +data "aws_subnets" "vpc" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +data "aws_kms_key" "kms_key" { + key_id = var.awskms_unseal_key_arn +} + +data "aws_iam_policy_document" "target" { + statement { + resources = ["*"] + + actions = [ + "ec2:DescribeInstances", + "secretsmanager:*" + ] + } + + statement { + resources = [var.awskms_unseal_key_arn] + + actions = [ + "kms:DescribeKey", + "kms:ListKeys", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey" + ] + } +} + +data "aws_iam_policy_document" "target_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "fleet" { + statement { + resources = ["*"] + + actions = [ + "ec2:DescribeImages", + "ec2:DescribeSubnets", + "ec2:RequestSpotInstances", + "ec2:TerminateInstances", + "ec2:DescribeInstanceStatus", + "ec2:CancelSpotFleetRequests", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:StartInstances", + "ec2:StopInstances", + ] + } + + statement { + effect = "Deny" + + resources = [ + "arn:aws:ec2:*:*:instance/*", + ] + + actions = [ + "ec2:RunInstances", + ] + + condition { + test = "StringNotEquals" + variable = "ec2:InstanceMarketType" + values = ["spot"] + } + } + + statement { + resources = ["*"] + + actions = [ + "iam:PassRole", + ] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = [ + "ec2.amazonaws.com", + ] + } + } + + statement { + resources = [ + "arn:aws:elasticloadbalancing:*:*:loadbalancer/*", + ] + + actions = [ + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + ] + } + + statement { + resources = [ + "arn:aws:elasticloadbalancing:*:*:*/*" + ] + + actions = [ + "elasticloadbalancing:RegisterTargets" + ] + } +} + +data "aws_iam_policy_document" "fleet_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["spotfleet.amazonaws.com"] + } + } +} + +data "enos_environment" "localhost" {} + +resource "random_string" "cluster_name" { + length = 8 + lower = true + upper = false + numeric = false + special = false +} + +resource "random_string" "unique_id" { + length = 4 + lower = true + upper = false + numeric = false + special = false +} + +locals { + instances = toset([for idx in range(var.instance_count) : tostring(idx)]) + cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result) + name_prefix = "${var.project_name}-${local.cluster_name}-${random_string.unique_id.result}" + fleet_tag = "${local.name_prefix}-spot-fleet-target" + fleet_tags = { + Name = "${local.name_prefix}-target" + Type = local.cluster_name + SpotFleet = local.fleet_tag + } +} + +resource "aws_iam_role" "target" { + name = "${local.name_prefix}-target-role" + assume_role_policy = data.aws_iam_policy_document.target_role.json +} + +resource "aws_iam_instance_profile" "target" { + name = "${local.name_prefix}-target-profile" + role = aws_iam_role.target.name +} + +resource "aws_iam_role_policy" "target" { + name = "${local.name_prefix}-target-policy" + role = aws_iam_role.target.id + policy = data.aws_iam_policy_document.target.json +} + +resource "aws_iam_role" "fleet" { + name = "${local.name_prefix}-fleet-role" + assume_role_policy = data.aws_iam_policy_document.fleet_role.json +} + +resource "aws_iam_role_policy" "fleet" { + name = "${local.name_prefix}-fleet-policy" + role = aws_iam_role.fleet.id + policy = data.aws_iam_policy_document.fleet.json +} + +resource "aws_security_group" "target" { + name = "${local.name_prefix}-target" + description = "Target instance security group" + vpc_id = var.vpc_id + + # SSH traffic + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [ + "${data.enos_environment.localhost.public_ip_address}/32", + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ] + } + + # Vault traffic + ingress { + from_port = 8200 + to_port = 8201 + protocol = "tcp" + cidr_blocks = flatten([ + "${data.enos_environment.localhost.public_ip_address}/32", + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + formatlist("%s/32", var.ssh_allow_ips) + ]) + } + + # Consul traffic + ingress { + from_port = 8301 + to_port = 8301 + protocol = "tcp" + cidr_blocks = [ + "${data.enos_environment.localhost.public_ip_address}/32", + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ] + } + + ingress { + from_port = 8301 + to_port = 8301 + protocol = "udp" + cidr_blocks = [ + "${data.enos_environment.localhost.public_ip_address}/32", + join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block), + ] + } + + # Internal traffic + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + } + + # External traffic + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = merge( + var.common_tags, + { + Name = "${local.name_prefix}-sg" + }, + ) +} + +resource "aws_launch_template" "target" { + name = "${local.name_prefix}-target" + image_id = var.ami_id + key_name = var.ssh_keypair + + iam_instance_profile { + name = aws_iam_instance_profile.target.name + } + + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.target.id] + } + + tag_specifications { + resource_type = "instance" + + tags = merge( + var.common_tags, + local.fleet_tags, + ) + } +} + +# There are three primary knobs we can turn to try and optimize our costs by +# using a spot fleet: our min and max instance requirements, our max bid +# price, and the allocation strategy to use when fulfilling the spot request. +# We've currently configured our instance requirements to allow for anywhere +# from 2-4 vCPUs and 4-16GB of RAM. We intentionally have a wide range +# to allow for a large instance size pool to be considered. Our next knob is our +# max bid price. As we're using spot fleets to save on instance cost, we never +# want to pay more for an instance than we were on-demand. We've set the max price +# to equal what we pay for t3.medium instances on-demand, which are the smallest +# reliable size for Vault scenarios. The final knob is the allocation strategy +# that AWS will use when looking for instances that meet our resource and cost +# requirements. We're using the "lowestPrice" strategy to get the absolute +# cheapest machines that will fit the requirements, but it comes with a slightly +# higher capacity risk than say, "capacityOptimized" or "priceCapacityOptimized". +# Unless we see capacity issues or instances being shut down then we ought to +# stick with that strategy. +resource "aws_spot_fleet_request" "targets" { + allocation_strategy = "lowestPrice" + fleet_type = "request" + iam_fleet_role = aws_iam_role.fleet.arn + // Set this to zero so re-runs don't plan for replacement + instance_pools_to_use_count = 0 + target_capacity = var.instance_count + terminate_instances_on_delete = true + wait_for_fulfillment = true + + launch_template_config { + launch_template_specification { + id = aws_launch_template.target.id + version = aws_launch_template.target.latest_version + } + + overrides { + spot_price = var.spot_price_max + subnet_id = data.aws_subnets.vpc.ids[0] + + instance_requirements { + burstable_performance = "included" + + memory_mib { + min = var.instance_mem_min + max = var.instance_mem_max + } + + vcpu_count { + min = var.instance_cpu_min + max = var.instance_cpu_max + } + } + } + } + + tags = merge( + var.common_tags, + local.fleet_tags, + ) +} + +data "aws_instances" "targets" { + depends_on = [ + aws_spot_fleet_request.targets, + ] + + instance_tags = local.fleet_tags + instance_state_names = [ + "pending", + "running", + ] + + filter { + name = "image-id" + values = [var.ami_id] + } + + filter { + name = "iam-instance-profile.arn" + values = [aws_iam_instance_profile.target.arn] + } +} + +data "aws_instance" "targets" { + depends_on = [ + aws_spot_fleet_request.targets, + data.aws_instances.targets + ] + for_each = local.instances + + instance_id = data.aws_instances.targets.ids[each.key] +} diff --git a/enos/modules/target_ec2_spot_fleet/outputs.tf b/enos/modules/target_ec2_spot_fleet/outputs.tf new file mode 100644 index 000000000000..2248388da521 --- /dev/null +++ b/enos/modules/target_ec2_spot_fleet/outputs.tf @@ -0,0 +1,11 @@ +output "cluster_name" { + value = local.cluster_name +} + +output "hosts" { + description = "The spot fleet target hosts" + value = { for idx in range(var.instance_count) : idx => { + public_ip = data.aws_instance.targets[idx].public_ip + private_ip = data.aws_instance.targets[idx].private_ip + } } +} diff --git a/enos/modules/target_ec2_spot_fleet/variables.tf b/enos/modules/target_ec2_spot_fleet/variables.tf new file mode 100644 index 000000000000..da41866554f8 --- /dev/null +++ b/enos/modules/target_ec2_spot_fleet/variables.tf @@ -0,0 +1,88 @@ +variable "ami_id" { + description = "The machine image identifier" + type = string +} + +variable "awskms_unseal_key_arn" { + type = string + description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key" + default = null +} + +variable "cluster_name" { + type = string + description = "A unique cluster identifier" + default = null +} + +variable "common_tags" { + description = "Common tags for cloud resources" + type = map(string) + default = { + Project = "Vault" + } +} + +variable "instance_mem_min" { + description = "The minimum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)" + type = number + default = 4096 // ~4 GB +} + +variable "instance_mem_max" { + description = "The maximum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)" + type = number + default = 16385 // ~16 GB +} + +variable "instance_cpu_min" { + description = "The minimum number of vCPU's for each instance in the fleet" + type = number + default = 2 +} + +variable "instance_cpu_max" { + description = "The maximum number of vCPU's for each instance in the fleet" + type = number + default = 8 // Unlikely we'll ever get that high due to spot price bid protection +} + +variable "instance_count" { + description = "The number of target instances to create" + type = number + default = 3 +} + +variable "instance_type" { + description = "Shim variable for target module variable compatibility that is not used. The spot fleet determines instance sizes" + type = string + default = null +} + +variable "project_name" { + description = "A unique project name" + type = string +} + +variable "spot_price_max" { + description = "The maximum hourly price to pay for each target instance" + type = string + // Current on-demand cost of linux t3.medium in us-east. + default = "0.0416" +} + +variable "ssh_allow_ips" { + description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted" + type = list(string) + default = [] +} + +variable "ssh_keypair" { + description = "SSH keypair used to connect to EC2 instances" + type = string +} + +variable "vpc_id" { + description = "The identifier of the VPC where the target instances will be created" + type = string +} diff --git a/enos/modules/vault_cluster/main.tf b/enos/modules/vault_cluster/main.tf new file mode 100644 index 000000000000..08455ca8f648 --- /dev/null +++ b/enos/modules/vault_cluster/main.tf @@ -0,0 +1,335 @@ +terraform { + required_providers { + # We need to specify the provider source in each module until we publish it + # to the public registry + enos = { + source = "app.terraform.io/hashicorp-qti/enos" + version = ">= 0.3.2" + } + } +} + +data "enos_environment" "localhost" {} + +locals { + bin_path = "${var.install_dir}/vault" + consul_bin_path = "${var.consul_install_dir}/consul" + key_shares = { + "awskms" = null + "shamir" = 5 + } + key_threshold = { + "awskms" = null + "shamir" = 3 + } + // In order to get Terraform to plan we have to use collections with keys + // that are known at plan time. In order for our module to work our var.target_hosts + // must be a map with known keys at plan time. Here we're creating locals + // that keep track of index values that point to our target hosts. + followers = toset(slice(local.instances, 1, length(local.instances))) + instances = [for idx in range(length(var.target_hosts)) : tostring(idx)] + leader = toset(slice(local.instances, 0, 1)) + recovery_shares = { + "awskms" = 5 + "shamir" = null + } + recovery_threshold = { + "awskms" = 3 + "shamir" = null + } + seal = { + "awskms" = { + type = "awskms" + attributes = { + kms_key_id = var.awskms_unseal_key_arn + } + } + "shamir" = { + type = "shamir" + attributes = null + } + } + storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ? + merge( + { + node_id = "${var.storage_node_prefix}_${idx}" + }, + var.storage_backend_addl_config + ) : + { + address = "127.0.0.1:8500" + path = "vault" + }) + ] +} + +resource "enos_remote_exec" "install_packages" { + for_each = { + for idx, host in var.target_hosts : idx => var.target_hosts[idx] + if length(var.packages) > 0 + } + + content = templatefile("${path.module}/templates/install-packages.sh", { + packages = join(" ", var.packages) + }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_bundle_install" "consul" { + for_each = { + for idx, host in var.target_hosts : idx => var.target_hosts[idx] + if var.storage_backend == "consul" + } + + destination = var.consul_install_dir + release = merge(var.consul_release, { product = "consul" }) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_bundle_install" "vault" { + for_each = var.target_hosts + + destination = var.install_dir + release = var.release == null ? var.release : merge({ product = "vault" }, var.release) + artifactory = var.artifactory_release + path = var.local_artifact_path + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_consul_start" "consul" { + for_each = enos_bundle_install.consul + + bin_path = local.consul_bin_path + data_dir = var.consul_data_dir + config = { + data_dir = var.consul_data_dir + datacenter = "dc1" + retry_join = ["provider=aws tag_key=Type tag_value=${var.consul_cluster_tag}"] + server = false + bootstrap_expect = 0 + log_level = "INFO" + log_file = var.consul_log_file + } + unit_name = "consul" + username = "consul" + + transport = { + ssh = { + host = var.target_hosts[each.key].public_ip + } + } +} + +resource "enos_vault_start" "leader" { + depends_on = [ + enos_consul_start.consul, + enos_bundle_install.vault, + ] + for_each = local.leader + + bin_path = local.bin_path + config_dir = var.config_dir + environment = var.config_env_vars + config = { + api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" + cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" + cluster_name = var.cluster_name + listener = { + type = "tcp" + attributes = { + address = "0.0.0.0:8200" + tls_disable = "true" + } + } + storage = { + type = var.storage_backend + attributes = ({ for key, value in local.storage_config[each.key] : key => value }) + } + seal = local.seal[var.unseal_method] + ui = true + } + license = var.license + manage_service = var.manage_service + username = "vault" + unit_name = "vault" + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_start" "followers" { + depends_on = [ + enos_vault_start.leader, + ] + for_each = local.followers + + bin_path = local.bin_path + config_dir = var.config_dir + environment = var.config_env_vars + config = { + api_addr = "http://${var.target_hosts[each.value].private_ip}:8200" + cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201" + cluster_name = var.cluster_name + listener = { + type = "tcp" + attributes = { + address = "0.0.0.0:8200" + tls_disable = "true" + } + } + storage = { + type = var.storage_backend + attributes = { for key, value in local.storage_config[each.key] : key => value } + } + seal = local.seal[var.unseal_method] + ui = true + } + license = var.license + manage_service = var.manage_service + username = "vault" + unit_name = "vault" + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_init" "leader" { + depends_on = [ + enos_vault_start.followers, + ] + for_each = toset([ + for idx, leader in local.leader : leader + if var.initialize_cluster + ]) + + bin_path = local.bin_path + vault_addr = enos_vault_start.leader[0].config.api_addr + + key_shares = local.key_shares[var.unseal_method] + key_threshold = local.key_threshold[var.unseal_method] + + recovery_shares = local.recovery_shares[var.unseal_method] + recovery_threshold = local.recovery_threshold[var.unseal_method] + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +resource "enos_vault_unseal" "leader" { + depends_on = [ + enos_vault_start.followers, + enos_vault_init.leader, + ] + for_each = enos_vault_init.leader // only unseal the leader if we initialized it + + bin_path = local.bin_path + vault_addr = enos_vault_start.leader[each.key].config.api_addr + seal_type = var.unseal_method + unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) + + transport = { + ssh = { + host = var.target_hosts[tolist(local.leader)[0]].public_ip + } + } +} + +resource "enos_vault_unseal" "followers" { + depends_on = [ + enos_vault_init.leader, + enos_vault_unseal.leader, + ] + // Only unseal followers if we're not using an auto-unseal method and we've + // initialized the cluster + for_each = toset([ + for idx, follower in local.followers : follower + if var.unseal_method == "shamir" && var.initialize_cluster + ]) + + bin_path = local.bin_path + vault_addr = enos_vault_start.followers[each.key].config.api_addr + seal_type = var.unseal_method + unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex) + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} + +// Force unseal the cluster. This is used if the vault-cluster module is used +// to add additional nodes to a cluster via auto-pilot, or some other means. +// When that happens we'll want to set initialize_cluster to false and +// force_unseal to true. +resource "enos_vault_unseal" "maybe_force_unseal" { + depends_on = [ + enos_vault_start.followers, + ] + for_each = { + for idx, host in var.target_hosts : idx => host + if var.force_unseal && !var.initialize_cluster + } + + bin_path = local.bin_path + vault_addr = "http://localhost:8200" + seal_type = var.unseal_method + unseal_keys = coalesce( + var.shamir_unseal_keys, + try(enos_vault_init.leader[0].unseal_keys_hex, null), + ) + + transport = { + ssh = { + host = each.value.public_ip + } + } +} + +resource "enos_remote_exec" "vault_write_license" { + for_each = toset([ + for idx, leader in local.leader : leader + if var.initialize_cluster + ]) + + depends_on = [ + enos_vault_unseal.leader, + enos_vault_unseal.maybe_force_unseal, + ] + + content = templatefile("${path.module}/templates/vault-write-license.sh", { + bin_path = local.bin_path, + root_token = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none") + license = coalesce(var.license, "none") + }) + + transport = { + ssh = { + host = var.target_hosts[each.value].public_ip + } + } +} diff --git a/enos/modules/vault_cluster/outputs.tf b/enos/modules/vault_cluster/outputs.tf new file mode 100644 index 000000000000..8e72ef44668b --- /dev/null +++ b/enos/modules/vault_cluster/outputs.tf @@ -0,0 +1,55 @@ +output "public_ips" { + description = "Vault cluster target host public_ips" + value = [for host in var.target_hosts : host.public_ip] +} + +output "private_ips" { + description = "Vault cluster target host private_ips" + value = [for host in var.target_hosts : host.private_ip] +} + +output "target_hosts" { + description = "The vault cluster instances that were created" + + value = var.target_hosts +} +output "root_token" { + value = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none") +} + +output "unseal_keys_b64" { + value = try(enos_vault_init.leader[0].unseal_keys_b64, []) +} + +output "unseal_keys_hex" { + value = try(enos_vault_init.leader[0].unseal_keys_hex, null) +} + +output "unseal_shares" { + value = try(enos_vault_init.leader[0].unseal_keys_shares, -1) +} + +output "unseal_threshold" { + value = try(enos_vault_init.leader[0].unseal_keys_threshold, -1) +} + +output "recovery_keys_b64" { + value = try(enos_vault_init.leader[0].recovery_keys_b64, []) +} + +output "recovery_keys_hex" { + value = try(enos_vault_init.leader[0].recovery_keys_hex, []) +} + +output "recovery_key_shares" { + value = try(enos_vault_init.leader[0].recovery_keys_shares, -1) +} + +output "recovery_threshold" { + value = try(enos_vault_init.leader[0].recovery_keys_threshold, -1) +} + +output "cluster_name" { + description = "The Vault cluster name" + value = var.cluster_name +} diff --git a/enos/modules/vault_cluster/templates/install-packages.sh b/enos/modules/vault_cluster/templates/install-packages.sh new file mode 100755 index 000000000000..61b6a1272d09 --- /dev/null +++ b/enos/modules/vault_cluster/templates/install-packages.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -ex -o pipefail + +packages="${packages}" + +if [ "$packages" == "" ] +then + echo "No dependencies to install." + exit 0 +fi + +function retry { + local retries=$1 + shift + local count=0 + + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + return "$exit" + fi + done + + return 0 +} + +echo "Installing Dependencies: $packages" +if [ -f /etc/debian_version ]; then + # Make sure cloud-init is not modifying our sources list while we're trying + # to install. + retry 7 grep ec2 /etc/apt/sources.list + + cd /tmp + retry 5 sudo apt update + retry 5 sudo apt install -y "$${packages[@]}" +else + cd /tmp + retry 7 sudo yum -y install "$${packages[@]}" +fi diff --git a/enos/modules/vault_cluster/templates/vault-write-license.sh b/enos/modules/vault_cluster/templates/vault-write-license.sh new file mode 100755 index 000000000000..10de84ba56f1 --- /dev/null +++ b/enos/modules/vault_cluster/templates/vault-write-license.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +license='${license}' +if test $license = "none"; then + exit 0 +fi + +function retry { + local retries=$1 + shift + local count=0 + + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + + if [ "$count" -lt "$retries" ]; then + sleep "$wait" + else + return "$exit" + fi + done + + return 0 +} + +export VAULT_ADDR=http://localhost:8200 +export VAULT_TOKEN='${root_token}' + +# Temporary hack until we can make the unseal resource handle legacy license +# setting. If we're running 1.8 and above then we shouldn't try to set a license. +ver=$(${bin_path} version) +if [[ "$(echo "$ver" |awk '{print $2}' |awk -F'.' '{print $2}')" -ge 8 ]]; then + exit 0 +fi + +retry 5 ${bin_path} write /sys/license text="$license" diff --git a/enos/modules/vault_cluster/variables.tf b/enos/modules/vault_cluster/variables.tf new file mode 100644 index 000000000000..d7be243e2efb --- /dev/null +++ b/enos/modules/vault_cluster/variables.tf @@ -0,0 +1,176 @@ +variable "artifactory_release" { + type = object({ + username = string + token = string + url = string + sha256 = string + }) + description = "The Artifactory release information to install Vault artifacts from Artifactory" + default = null +} + +variable "awskms_unseal_key_arn" { + type = string + description = "The AWSKMS key ARN if using the awskms unseal method" + default = null +} + +variable "cluster_name" { + type = string + description = "The Vault cluster name" + default = null +} + +variable "config_dir" { + type = string + description = "The directory to use for Vault configuration" + default = "/etc/vault.d" +} + +variable "config_env_vars" { + description = "Optional Vault configuration environment variables to set starting Vault" + type = map(string) + default = null +} + +variable "consul_cluster_tag" { + type = string + description = "The retry_join tag to use for Consul" + default = null +} + +variable "consul_data_dir" { + type = string + description = "The directory where the consul will store data" + default = "/opt/consul/data" +} + +variable "consul_install_dir" { + type = string + description = "The directory where the consul binary will be installed" + default = "/opt/consul/bin" +} + +variable "consul_log_file" { + type = string + description = "The file where the consul will write log output" + default = "/var/log/consul.log" +} + +variable "consul_release" { + type = object({ + version = string + edition = string + }) + description = "Consul release version and edition to install from releases.hashicorp.com" + default = { + version = "1.15.1" + edition = "oss" + } +} + +variable "force_unseal" { + type = bool + description = "Always unseal the Vault cluster even if we're not initializing it" + default = false +} + +variable "initialize_cluster" { + type = bool + description = "Initialize the Vault cluster" + default = true +} + +variable "install_dir" { + type = string + description = "The directory where the vault binary will be installed" + default = "/opt/vault/bin" +} + +variable "license" { + type = string + sensitive = true + description = "The value of the Vault license" + default = null +} + +variable "local_artifact_path" { + type = string + description = "The path to a locally built vault artifact to install. It can be a zip archive, RPM, or Debian package" + default = null +} + +variable "manage_service" { + type = bool + description = "Manage the Vault service users and systemd unit. Disable this to use configuration in RPM and Debian packages" + default = true +} + +variable "packages" { + type = list(string) + description = "A list of packages to install via the target host package manager" + default = [] +} + +variable "release" { + type = object({ + version = string + edition = string + }) + description = "Vault release version and edition to install from releases.hashicorp.com" + default = null +} + +variable "root_token" { + type = string + description = "The Vault root token that we can use to intialize and configure the cluster" + default = null +} + +variable "shamir_unseal_keys" { + type = list(string) + description = "Shamir unseal keys. Often only used adding additional nodes to an already initialized cluster." + default = null +} + +variable "storage_backend" { + type = string + description = "The storage backend to use" + default = "raft" + + validation { + condition = contains(["raft", "consul"], var.storage_backend) + error_message = "The storage_backend must be either raft or consul. No other storage backends are supported." + } +} + +variable "storage_backend_addl_config" { + type = map(any) + description = "An optional set of key value pairs to inject into the storage block" + default = {} +} + +variable "storage_node_prefix" { + type = string + description = "A prefix to use for each node in the Vault storage configuration" + default = "node" +} + +variable "target_hosts" { + description = "The target machines host addresses to use for the Vault cluster" + type = map(object({ + private_ip = string + public_ip = string + })) +} + +variable "unseal_method" { + type = string + description = "The method by which to unseal the Vault cluster" + default = "awskms" + + validation { + condition = contains(["awskms", "shamir"], var.unseal_method) + error_message = "The unseal_method must be either awskms or shamir. No other unseal methods are supported." + } +} From e3862c51e019945030a74db0df76e51c742e49f2 Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Fri, 7 Apr 2023 13:40:48 -0600 Subject: [PATCH 2/2] use us-east-1 instead of us-west-1 because it's cheaper Signed-off-by: Ryan Cragun --- .../build-github-oss-linux-amd64-zip.json | 10 +++++----- .../build-github-oss-linux-arm64-zip.json | 10 +++++----- ...se_testing_oss-artifactory-oss-linux-amd64-zip.json | 10 +++++----- ...se_testing_oss-artifactory-oss-linux-arm64-zip.json | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json b/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json index ab09a413bad3..80b3d55212b2 100644 --- a/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json +++ b/.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json @@ -2,7 +2,7 @@ "include": [ { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 3 }, { @@ -12,7 +12,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { @@ -22,7 +22,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -32,7 +32,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 5 }, { @@ -42,7 +42,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { diff --git a/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json b/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json index ec951fdd0a18..a497fb0ebe00 100644 --- a/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json +++ b/.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json @@ -7,7 +7,7 @@ }, { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -17,7 +17,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 4 }, { @@ -27,7 +27,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { @@ -37,7 +37,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 3 }, { @@ -47,7 +47,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 5 } ] diff --git a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json index 70e5ea1c3c24..857677b72f07 100644 --- a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json +++ b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json @@ -2,7 +2,7 @@ "include": [ { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -12,7 +12,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -22,7 +22,7 @@ }, { "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -32,7 +32,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -42,7 +42,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { diff --git a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json index e6e9edb10f28..1c67cd3bcfdb 100644 --- a/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json +++ b/.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json @@ -7,17 +7,17 @@ }, { "scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { "scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, { "scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -27,7 +27,7 @@ }, { "scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 2 }, { @@ -42,7 +42,7 @@ }, { "scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle", - "aws_region": "us-west-1", + "aws_region": "us-east-1", "test_group": 1 }, {