backport of commit 27ab988 (#26584)

Co-authored-by: Ryan Cragun <me@ryan.ec>

enos/enos-globals.hcl

@@ -14,6 +14,7 @@ globals {
     "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
     "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
   }
+  config_modes    = ["env", "file"]
   consul_versions = ["1.14.11", "1.15.7", "1.16.3", "1.17.0"]
   distros         = ["ubuntu", "rhel"]
   distro_version = {

enos/enos-modules.hcl

@@ -183,6 +183,12 @@ module "vault_setup_perf_secondary" {
   vault_install_dir = var.vault_install_dir
 }
 
+module "vault_step_down" {
+  source = "./modules/vault_step_down"
+
+  vault_install_dir = var.vault_install_dir
+}
+
 module "vault_test_ui" {
   source = "./modules/vault_test_ui"
 
@@ -203,7 +209,6 @@ module "vault_upgrade" {
   vault_instance_count = var.vault_instance_count
 }
 
-
 module "vault_verify_autopilot" {
   source = "./modules/vault_verify_autopilot"
 

enos/enos-scenario-agent.hcl

@@ -7,11 +7,11 @@ scenario "agent" {
     artifact_source = global.artifact_sources
     artifact_type   = global.artifact_types
     backend         = global.backends
+    config_mode     = global.config_modes
     consul_version  = global.consul_versions
     distro          = global.distros
     edition         = global.editions
     seal            = global.seals
-    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -195,6 +195,7 @@ scenario "agent" {
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       consul_release = matrix.backend == "consul" ? {
         edition = var.backend_edition
@@ -207,7 +208,6 @@ scenario "agent" {
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
       seal_attributes      = step.create_seal_key.attributes
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts

enos/enos-scenario-autopilot.hcl

@@ -6,11 +6,11 @@ scenario "autopilot" {
     arch            = global.archs
     artifact_source = global.artifact_sources
     artifact_type   = global.artifact_types
+    config_mode     = global.config_modes
     distro          = global.distros
     edition         = global.editions
     initial_version = global.upgrade_initial_versions
     seal            = global.seals
-    seal_ha_beta    = ["true", "false"]
 
     # Autopilot wasn't available before 1.11.x
     exclude {
@@ -157,6 +157,7 @@ scenario "autopilot" {
 
     variables {
       cluster_name         = step.create_vault_cluster_targets.cluster_name
+      config_mode          = matrix.config_mode
       enable_audit_devices = var.vault_enable_audit_devices
       install_dir          = local.vault_install_dir
       license              = matrix.edition != "ce" ? step.read_license.license : null
@@ -166,7 +167,6 @@ scenario "autopilot" {
         version = matrix.initial_version
       }
       seal_attributes = step.create_seal_key.attributes
-      seal_ha_beta    = matrix.seal_ha_beta
       seal_type       = matrix.seal
       storage_backend = "raft"
       storage_backend_addl_config = {
@@ -241,6 +241,7 @@ scenario "autopilot" {
       artifactory_release         = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
       enable_audit_devices        = var.vault_enable_audit_devices
       cluster_name                = step.create_vault_cluster_targets.cluster_name
+      config_mode                 = matrix.config_mode
       log_level                   = var.vault_log_level
       force_unseal                = matrix.seal == "shamir"
       initialize_cluster          = false
@@ -250,7 +251,6 @@ scenario "autopilot" {
       manage_service              = local.manage_service
       packages                    = concat(global.packages, global.distro_packages[matrix.distro])
       root_token                  = step.create_vault_cluster.root_token
-      seal_ha_beta                = matrix.seal_ha_beta
       seal_attributes             = step.create_seal_key.attributes
       seal_type                   = matrix.seal
       shamir_unseal_keys          = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null

enos/enos-scenario-proxy.hcl

@@ -7,11 +7,11 @@ scenario "proxy" {
     artifact_source = global.artifact_sources
     artifact_type   = global.artifact_types
     backend         = global.backends
+    config_mode     = global.config_modes
     consul_version  = global.consul_versions
     distro          = global.distros
     edition         = global.editions
     seal            = global.seals
-    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -195,6 +195,7 @@ scenario "proxy" {
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       consul_release = matrix.backend == "consul" ? {
         edition = var.backend_edition
@@ -206,7 +207,6 @@ scenario "proxy" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_attributes      = step.create_seal_key.attributes
       seal_type            = matrix.seal
       storage_backend      = matrix.backend

enos/enos-scenario-replication.hcl

@@ -9,12 +9,12 @@ scenario "replication" {
     arch              = global.archs
     artifact_source   = global.artifact_sources
     artifact_type     = global.artifact_types
+    config_mode       = global.config_modes
     consul_version    = global.consul_versions
     distro            = global.distros
     edition           = global.editions
     primary_backend   = global.backends
     primary_seal      = global.seals
-    seal_ha_beta      = ["true", "false"]
     secondary_backend = global.backends
     secondary_seal    = global.seals
 
@@ -280,6 +280,7 @@ scenario "replication" {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
       backend_cluster_name    = step.create_primary_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       cluster_name            = step.create_primary_cluster_targets.cluster_name
       consul_release = matrix.primary_backend == "consul" ? {
@@ -293,7 +294,6 @@ scenario "replication" {
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
       seal_attributes      = step.create_primary_seal_key.attributes
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_type            = matrix.primary_seal
       storage_backend      = matrix.primary_backend
       target_hosts         = step.create_primary_cluster_targets.hosts
@@ -338,6 +338,7 @@ scenario "replication" {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
       backend_cluster_name    = step.create_secondary_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.secondary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       cluster_name            = step.create_secondary_cluster_targets.cluster_name
       consul_release = matrix.secondary_backend == "consul" ? {
@@ -351,7 +352,6 @@ scenario "replication" {
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
       seal_attributes      = step.create_secondary_seal_key.attributes
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_type            = matrix.secondary_seal
       storage_backend      = matrix.secondary_backend
       target_hosts         = step.create_secondary_cluster_targets.hosts
@@ -632,6 +632,7 @@ scenario "replication" {
       backend_cluster_name    = step.create_primary_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_primary_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       consul_release = matrix.primary_backend == "consul" ? {
         edition = var.backend_edition
@@ -646,7 +647,6 @@ scenario "replication" {
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
       root_token           = step.create_primary_cluster.root_token
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_attributes      = step.create_primary_seal_key.attributes
       seal_type            = matrix.primary_seal
       shamir_unseal_keys   = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null

enos/enos-scenario-seal-ha.hcl

@@ -7,6 +7,7 @@ scenario "seal_ha" {
     artifact_source = global.artifact_sources
     artifact_type   = global.artifact_types
     backend         = global.backends
+    config_mode     = global.config_modes
     consul_version  = global.consul_versions
     distro          = global.distros
     edition         = global.editions
@@ -218,6 +219,7 @@ scenario "seal_ha" {
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       consul_release = matrix.backend == "consul" ? {
         edition = var.backend_edition
@@ -365,7 +367,7 @@ scenario "seal_ha" {
   }
 
   // Wait for our cluster to elect a leader
-  step "wait_for_new_leader" {
+  step "wait_for_leader_election" {
     module     = module.vault_wait_for_leader
     depends_on = [step.add_ha_seal_to_cluster]
 
@@ -381,6 +383,54 @@ scenario "seal_ha" {
     }
   }
 
+  step "get_leader_ip_for_step_down" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_leader_election]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Force a step down to trigger a new leader election
+  step "vault_leader_step_down" {
+    module     = module.vault_step_down
+    depends_on = [step.get_leader_ip_for_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      leader_host       = step.get_leader_ip_for_step_down.leader_host
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_new_leader" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.vault_leader_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
   step "get_updated_cluster_ips" {
     module     = module.vault_get_cluster_ips
     depends_on = [step.wait_for_new_leader]

enos/enos-scenario-smoke.hcl

@@ -7,11 +7,11 @@ scenario "smoke" {
     artifact_source = global.artifact_sources
     artifact_type   = global.artifact_types
     backend         = global.backends
+    config_mode     = global.config_modes
     consul_version  = global.consul_versions
     distro          = global.distros
     edition         = global.editions
     seal            = global.seals
-    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -195,6 +195,7 @@ scenario "smoke" {
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
       consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
       consul_release = matrix.backend == "consul" ? {
         edition = var.backend_edition
@@ -206,7 +207,6 @@ scenario "smoke" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_attributes      = step.create_seal_key.attributes
       seal_type            = matrix.seal
       storage_backend      = matrix.backend
@@ -215,7 +215,7 @@ scenario "smoke" {
   }
 
   // Wait for our cluster to elect a leader
-  step "wait_for_leader" {
+  step "wait_for_new_leader" {
     module     = module.vault_wait_for_leader
     depends_on = [step.create_vault_cluster]
 
@@ -231,6 +231,54 @@ scenario "smoke" {
     }
   }
 
+  step "get_leader_ip_for_step_down" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_new_leader]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Force a step down to trigger a new leader election
+  step "vault_leader_step_down" {
+    module     = module.vault_step_down
+    depends_on = [step.get_leader_ip_for_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      leader_host       = step.get_leader_ip_for_step_down.leader_host
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_leader" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.vault_leader_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
   step "get_vault_cluster_ips" {
     module     = module.vault_get_cluster_ips
     depends_on = [step.wait_for_leader]

enos/enos-scenario-ui.hcl

@@ -3,9 +3,8 @@
 
 scenario "ui" {
   matrix {
-    backend      = global.backends
-    edition      = ["ce", "ent"]
-    seal_ha_beta = ["true", "false"]
+    backend = global.backends
+    edition = ["ce", "ent"]
   }
 
   terraform_cli = terraform_cli.default
@@ -180,7 +179,6 @@ scenario "ui" {
       license              = matrix.edition != "ce" ? step.read_vault_license.license : null
       local_artifact_path  = local.bundle_path
       packages             = global.distro_packages["ubuntu"]
-      seal_ha_beta         = matrix.seal_ha_beta
       seal_name            = step.create_seal_key.resource_name
       seal_type            = local.seal
       storage_backend      = matrix.backend