Add initial Consul integration

Should have Nomad and Consul deployed and configured with mTLS. ACLs are currently not enabled on Consul, only Nomad.

This should provide the minimal working example using mTLS to get the cought dashboard working after a ton of tinkering. 😭

The links I used during my investigation/debugging session:
* hashicorp/nomad#6463
* https://learn.hashicorp.com/nomad/consul-integration/nomad-connect-acl#run-a-connect-enabled-job
* hashicorp/nomad#6594
* hashicorp/nomad#4276
hashicorp/nomad#7715
* https://www.consul.io/docs/agent/options
⭐ * hashicorp/nomad#7602

Makefile

@@ -69,4 +69,28 @@ terraform/destroy/example: ## Runs and auto-apporves the Terraform destroy comma
 	cd example && terraform destroy \
 		-auto-approve \
 		-var="project=${GOOGLE_PROJECT}" \
-		-var="credentials=${GOOGLE_APPLICATION_CREDENTIALS}"
+		-var="credentials=${GOOGLE_APPLICATION_CREDENTIALS}"
+
+.PHONY: ssh/client
+ssh/client: ## Connects to the client instance using SSH
+	gcloud compute ssh client-0 --tunnel-through-iap
+
+.PHONY: ssh/server
+ssh/server: ## Connects to the server instance using SSH
+	gcloud compute ssh server-0 --tunnel-through-iap
+
+.PHONY: ssh/proxy/consul
+ssh/proxy/consul: ## Forwards the Consul server port to localhost
+	gcloud compute ssh server-0 --tunnel-through-iap -- -f -N -L 127.0.0.1:8500:127.0.0.1:8500
+
+.PHONY: ssh/proxy/nomad
+ssh/proxy/nomad: ## Forwards the Nomad server port to localhost
+	gcloud compute ssh server-0 --tunnel-through-iap -- -f -N -L 127.0.0.1:4646:127.0.0.1:4646
+
+.PHONY: ssh/proxy/nomad
+ssh/proxy/mtls/nomad: ## Forwards the Nomad server port to localhost, using the custom mTLS terminating proxy script
+	go run ssh-mtls-terminating-proxy.go
+
+.PHONY: ssh/proxy/count-dashboard
+ssh/proxy/count-dashboard: ## Forwards the example dashboard serverive port to localhost
+	gcloud compute ssh client-0 --tunnel-through-iap -- -f -N -L 127.0.0.1:9002:0.0.0.0:9002

README.md

@@ -14,6 +14,7 @@
 * Only the [Docker task driver](https://www.nomadproject.io/docs/drivers/docker) is enabled by default.
 * Runs the Docker daemon with `no-new-privileges=true` and `icc=false` set by default.
 * Installs the [gVisor](https://gvisor.dev/) container runtime by default (`runsc`).
+* Installs HashiCorp's [Consul](https://www.consul.io/) service mesh.
 
 ## Cloud Shell Interactive Tutorial
 
@@ -80,7 +81,7 @@ $ gcloud logging read 'resource.type="gce_instance" jsonPayload.ident="nomad"'
 ```
 
 ```console
-$ gcloud logging read 'resource.type="gce_instance" jsonPayload.ident="nomad" jsonPayload.host="nomad-server-0"' --format=json | jq -r '.[] | .jsonPayload.message' | less
+$ gcloud logging read 'resource.type="gce_instance" jsonPayload.ident="nomad" jsonPayload.host="server-0"' --format=json | jq -r '.[] | .jsonPayload.message' | less
 ...
 ```
 

consul_tls_ca.tf

@@ -0,0 +1,23 @@
+resource "tls_private_key" "consul-ca" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_self_signed_cert" "consul-ca" {
+  is_ca_certificate     = true
+  validity_period_hours = 87600
+
+  key_algorithm   = tls_private_key.consul-ca.algorithm
+  private_key_pem = tls_private_key.consul-ca.private_key_pem
+
+  subject {
+    common_name  = "consul-ca.local"
+    organization = var.tls_organization
+  }
+
+  allowed_uses = [
+    "cert_signing",
+    "digital_signature",
+    "key_encipherment",
+  ]
+}

consul_tls_cli.tf

@@ -0,0 +1,34 @@
+resource "tls_private_key" "consul-cli" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "consul-cli" {
+  key_algorithm   = tls_private_key.consul-cli.algorithm
+  private_key_pem = tls_private_key.consul-cli.private_key_pem
+
+  dns_names = [
+    "localhost",
+    "cli.dc1.consul",
+  ]
+
+  subject {
+    common_name  = "client.global.consul"
+    organization = var.tls_organization
+  }
+}
+
+resource "tls_locally_signed_cert" "consul-cli" {
+  cert_request_pem = tls_cert_request.consul-cli.cert_request_pem
+
+  ca_key_algorithm   = tls_private_key.consul-ca.algorithm
+  ca_private_key_pem = tls_private_key.consul-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.consul-ca.cert_pem
+
+  validity_period_hours = 87600
+
+  allowed_uses = [
+    "digital_signature",
+    "key_encipherment",
+  ]
+}

consul_tls_client.tf

@@ -0,0 +1,40 @@
+# TODO(kent): delete once auto_encrypt is setup/verified
+
+resource "tls_private_key" "consul-client" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "consul-client" {
+  key_algorithm   = tls_private_key.consul-client.algorithm
+  private_key_pem = tls_private_key.consul-client.private_key_pem
+
+  ip_addresses = [
+    "127.0.0.1",
+  ]
+
+  dns_names = [
+    "localhost",
+    "client.dc1.consul",
+  ]
+
+  subject {
+    common_name  = "client.dc1.consul"
+    organization = var.tls_organization
+  }
+}
+
+resource "tls_locally_signed_cert" "consul-client" {
+  cert_request_pem = tls_cert_request.consul-client.cert_request_pem
+
+  ca_key_algorithm   = tls_private_key.consul-ca.algorithm
+  ca_private_key_pem = tls_private_key.consul-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.consul-ca.cert_pem
+
+  validity_period_hours = 87600
+
+  allowed_uses = [
+    "server_auth",
+    "client_auth",
+  ]
+}

consul_tls_server.tf

@@ -0,0 +1,38 @@
+resource "tls_private_key" "consul-server" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "consul-server" {
+  key_algorithm   = tls_private_key.consul-server.algorithm
+  private_key_pem = tls_private_key.consul-server.private_key_pem
+
+  ip_addresses = [
+    "127.0.0.1",
+  ]
+
+  dns_names = [
+    "localhost",
+    "server.dc1.consul",
+  ]
+
+  subject {
+    common_name  = "server.dc1.consul"
+    organization = var.tls_organization
+  }
+}
+
+resource "tls_locally_signed_cert" "consul-server" {
+  cert_request_pem = tls_cert_request.consul-server.cert_request_pem
+
+  ca_key_algorithm   = tls_private_key.consul-ca.algorithm
+  ca_private_key_pem = tls_private_key.consul-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.consul-ca.cert_pem
+
+  validity_period_hours = 87600
+
+  allowed_uses = [
+    "server_auth",
+    "client_auth",
+  ]
+}

gossip_keys.tf

@@ -0,0 +1,9 @@
+resource "random_password" "nomad-gossip-key" {
+  length  = 16
+  special = true
+}
+
+resource "random_password" "consul-gossip-key" {
+  length  = 16
+  special = true
+}

load_balancer.tf

@@ -3,8 +3,8 @@ module "load_balancer" {
   region         = var.region
   name           = "load-balancer"
   service_port   = 4646
-  target_tags    = ["nomad-server"]
-  network        = module.nomad-network.name
-  instances      = formatlist("${format("%s-%s/nomad-server", var.region, var.zone)}-%d", range(var.server_instances))
-  and_depends_on = [module.nomad-network]
+  target_tags    = ["server"]
+  network        = module.network.name
+  instances      = formatlist("${format("%s-%s/server", var.region, var.zone)}-%d", range(var.server_instances))
+  and_depends_on = [module.network]
 }

modules/network/network.tf

@@ -1,10 +1,10 @@
-resource "google_compute_network" "nomad" {
+resource "google_compute_network" "default" {
   name = var.name
   auto_create_subnetworks = false
 }
 
-resource "google_compute_subnetwork" "nomad" {
-  network = google_compute_network.nomad.name
+resource "google_compute_subnetwork" "default" {
+  network = google_compute_network.default.name
   name    = var.name
   region  = var.region
 
@@ -13,24 +13,24 @@ resource "google_compute_subnetwork" "nomad" {
 
 resource "google_compute_firewall" "allow_icmp" {
   name    = "allow-icmp"
-  network = google_compute_network.nomad.name
+  network = google_compute_network.default.name
 
   allow {
     protocol = "icmp"
   }
 }
 
-module "nomad-ssh" {
+module "ssh" {
     source      = "../open-port"
-    network     = google_compute_network.nomad.name
-    name        = "nomad-ssh"
+    network     = google_compute_network.default.name
+    name        = "allow-all-ssh"
     port        = 22
     protocol    = "tcp"
 }
 
 module "nomad-http" {
     source      = "../open-port"
-    network     = google_compute_network.nomad.name
+    network     = google_compute_network.default.name
     name        = "nomad-http"
     port        = 4646
     protocol    = "tcp"
@@ -39,7 +39,7 @@ module "nomad-http" {
 
 module "nomad-rpc" {
     source      = "../open-port"
-    network     = google_compute_network.nomad.name
+    network     = google_compute_network.default.name
     name        = "nomad-rpc"
     port        = 4647
     protocol    = "tcp"
@@ -48,7 +48,7 @@ module "nomad-rpc" {
 
 module "nomad-wan-tcp" {
     source      = "../open-port"
-    network     = google_compute_network.nomad.name
+    network     = google_compute_network.default.name
     name        = "nomad-wan-tcp"
     port        = 4648
     protocol    = "tcp"
@@ -57,9 +57,112 @@ module "nomad-wan-tcp" {
 
 module "nomad-wan-udp" {
     source      = "../open-port"
-    network     = google_compute_network.nomad.name
+    network     = google_compute_network.default.name
     name        = "nomad-wan-udp"
     port        = 4648
     protocol    = "udp"
     source_tags = var.gossip_source_tags
 }
+
+module "consul-http" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-http"
+    port        = 8500
+    protocol    = "tcp"
+    source_tags = var.http_source_tags
+}
+
+module "consul-https" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-https"
+    port        = 8501
+    protocol    = "tcp"
+    source_tags = var.http_source_tags
+}
+
+module "consul-grpc" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-grpc"
+    port        = 8502
+    protocol    = "tcp"
+    source_tags = var.http_source_tags
+}
+
+module "consul-dns-tcp" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-dns-tcp"
+    port        = 8600
+    protocol    = "tcp"
+    source_tags = var.dns_source_tags
+}
+
+module "consul-dns-udp" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-dns-udp"
+    port        = 8600
+    protocol    = "udp"
+    source_tags = var.dns_source_tags
+}
+
+module "consul-rpc" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-rpc"
+    port        = 8300
+    protocol    = "tcp"
+    source_tags = var.rpc_source_tags
+}
+
+module "consul-serf-wan-tcp" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-serf-wan-tcp"
+    port        = 8301
+    protocol    = "tcp"
+    source_tags = var.gossip_source_tags
+}
+
+module "consul-serf-wan-udp" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-serf-wan-udp"
+    port        = 8302
+    protocol    = "udp"
+    source_tags = var.gossip_source_tags
+}
+
+module "consul-serf-lan-tcp" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-serf-lan-tcp"
+    port        = 8301
+    protocol    = "tcp"
+    source_tags = var.gossip_source_tags
+}
+
+module "consul-serf-lan-udp" {
+    source      = "../open-port"
+    network     = google_compute_network.default.name
+    name        = "consul-serf-lan-udp"
+    port        = 8301
+    protocol    = "udp"
+    source_tags = var.gossip_source_tags
+}
+
+resource "google_compute_firewall" "allow-all-internal-dyanmic-ports" {
+  name        = "allow-all-internal-dynamic-ports"
+  network     = google_compute_network.default.name
+  source_tags = var.gossip_source_tags
+
+  # https://www.nomadproject.io/docs/job-specification/network#dynamic-ports
+  # https://www.consul.io/docs/install/ports#ports-table
+  allow {
+    protocol = "tcp"
+    ports    = ["0-65535"]
+  }
+}