E2E: use remote-exec via TF0.14.7+

The E2E provisioning used local-exec to call ssh in a for loop in a hacky
workaround hashicorp/terraform#25634, which
prevented remote-exec from working on Windows. Move to a newer version of
Terraform that fixes the remote-exec bug to make provisioning more reliable
and observable.

Note that Windows remote-exec needs to include the `powershell` call itself,
unlike Unix-alike remote-exec.

e2e/terraform/README.md

@@ -9,7 +9,7 @@ cluster along with configuration files for Nomad, Consul, and Vault.
 
 ## Setup
 
-You'll need Terraform 0.13+, as well as AWS credentials to create the Nomad
+You'll need Terraform 0.14.7+, as well as AWS credentials to create the Nomad
 cluster. This Terraform stack assumes that an appropriate instance role has
 been configured elsewhere and that you have the ability to `AssumeRole` into
 the AWS account.
@@ -111,9 +111,9 @@ so that's safely skipped.
 After deploying the infrastructure, you can get connection information
 about the cluster:
 
-- `$(terraform output environment)` will set your current shell's
-  `NOMAD_ADDR` and `CONSUL_HTTP_ADDR` to point to one of the cluster's
-  server nodes, and set the `NOMAD_E2E` variable.
+- `$(terraform output --raw environment)` will set your current shell's
+  `NOMAD_ADDR` and `CONSUL_HTTP_ADDR` to point to one of the cluster's server
+  nodes, and set the `NOMAD_E2E` variable.
 - `terraform output servers` will output the list of server node IPs.
 - `terraform output linux_clients` will output the list of Linux
   client node IPs.

e2e/terraform/provision-nomad/main.tf

@@ -1,13 +1,13 @@
 locals {
-  provision_script = var.platform == "windows_amd64" ? "C:/opt/provision.ps1" : "/opt/provision.sh"
+  provision_script = var.platform == "windows_amd64" ? "powershell C:/opt/provision.ps1" : "/opt/provision.sh"
 
   config_path = dirname("${path.root}/config/")
 
   config_files = compact(setunion(
     fileset(local.config_path, "**"),
   ))
 
-  update_config_command = var.platform == "windows_amd64" ? "if (test-path /opt/config) { Remove-Item -Path /opt/config -Force -Recurse }; cp -r /tmp/config /opt/config" : "sudo rm -rf /opt/config; sudo mv /tmp/config /opt/config"
+  update_config_command = var.platform == "windows_amd64" ? "powershell -Command \"& { if (test-path /opt/config) { Remove-Item -Path /opt/config -Force -Recurse }; cp -r C:/tmp/config /opt/config }\"" : "sudo rm -rf /opt/config; sudo mv /tmp/config /opt/config"
 
   # abstract-away platform-specific parameter expectations
   _arg = var.platform == "windows_amd64" ? "-" : "--"
@@ -25,17 +25,19 @@ resource "null_resource" "provision_nomad" {
     script = data.template_file.provision_script.rendered
   }
 
-  # Run the provisioner as a local-exec'd ssh command as a workaround for
-  # Windows remote-exec zero-byte scripts bug:
-  # https://github.com/hashicorp/terraform/issues/25634
-  # https://github.com/hashicorp/terraform/blob/master/CHANGELOG.md#0150-unreleased
-  #
-  # The retry behavior and explicit PasswordAuthenticaiton flag here are to
-  # workaround a race with the Windows userdata script that installs the
-  # authorized_key. Unfortunately this still results in a bunch of "permission
-  # denied" errors while waiting for those keys to be configured.
-  provisioner "local-exec" {
-    command = "until ssh -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${var.connection.private_key} -p ${var.connection.port} ${var.connection.user}@${var.connection.host} ${data.template_file.provision_script.rendered}; do sleep 5; done"
+
+  connection {
+    type            = "ssh"
+    user            = var.connection.user
+    host            = var.connection.host
+    port            = var.connection.port
+    private_key     = file(var.connection.private_key)
+    target_platform = var.platform == "windows_amd64" ? "windows" : "unix"
+    timeout         = "15m"
+  }
+
+  provisioner "remote-exec" {
+    inline = [data.template_file.provision_script.rendered]
   }
 
 }
@@ -93,12 +95,13 @@ resource "null_resource" "upload_nomad_binary" {
   }
 
   connection {
-    type        = "ssh"
-    user        = var.connection.user
-    host        = var.connection.host
-    port        = var.connection.port
-    private_key = file(var.connection.private_key)
-    timeout     = "15m"
+    type            = "ssh"
+    user            = var.connection.user
+    host            = var.connection.host
+    port            = var.connection.port
+    private_key     = file(var.connection.private_key)
+    target_platform = var.platform == "windows_amd64" ? "windows" : "unix"
+    timeout         = "15m"
   }
 
   provisioner "file" {
@@ -114,21 +117,22 @@ resource "null_resource" "upload_configs" {
   }
 
   connection {
-    type        = "ssh"
-    user        = var.connection.user
-    host        = var.connection.host
-    port        = var.connection.port
-    private_key = file(var.connection.private_key)
-    timeout     = "15m"
+    type            = "ssh"
+    user            = var.connection.user
+    host            = var.connection.host
+    port            = var.connection.port
+    private_key     = file(var.connection.private_key)
+    target_platform = var.platform == "windows_amd64" ? "windows" : "unix"
+    timeout         = "15m"
   }
 
   provisioner "file" {
     source      = local.config_path
     destination = "/tmp/"
   }
 
-  provisioner "local-exec" {
-    command = "until ssh -o PasswordAuthentication=no -o KbdInteractiveAuthentication=no -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${var.connection.private_key} -p ${var.connection.port} ${var.connection.user}@${var.connection.host} '${local.update_config_command}'; do sleep 5; done"
+  provisioner "remote-exec" {
+    inline = [local.update_config_command]
   }
 
 }