From b0e5626ee799a9e54aed3b8571e72592b206bcc6 Mon Sep 17 00:00:00 2001
From: staebler <staebler@redhat.com>
Date: Tue, 26 Mar 2019 13:13:20 -0400
Subject: [PATCH 1/4] upi/vsphere: add dns for control plane and compute
 hostnames

Add DNS records for the control plane and compute hostnames so that
they are resolvable from within the cluster.
---
 upi/vsphere/main.tf              |  1 +
 upi/vsphere/route53/main.tf      | 20 ++++++++++++++++++++
 upi/vsphere/route53/variables.tf |  4 ++++
 upi/vsphere/variables.tf         |  5 +++++
 4 files changed, 30 insertions(+)

diff --git a/upi/vsphere/main.tf b/upi/vsphere/main.tf
index aeca9abb616..282414f34b4 100644
--- a/upi/vsphere/main.tf
+++ b/upi/vsphere/main.tf
@@ -75,4 +75,5 @@ module "dns" {
   cluster_domain    = "${var.cluster_domain}"
   bootstrap_ip      = "${var.bootstrap_complete ? "" : var.bootstrap_ip}"
   control_plane_ips = "${var.control_plane_ips}"
+  compute_ips       = "${var.compute_ips}"
 }
diff --git a/upi/vsphere/route53/main.tf b/upi/vsphere/route53/main.tf
index bcf89381fde..3a6932ece15 100644
--- a/upi/vsphere/route53/main.tf
+++ b/upi/vsphere/route53/main.tf
@@ -71,3 +71,23 @@ resource "aws_route53_record" "ingress" {
   name    = "*.apps.${var.cluster_domain}"
   records = ["${var.compute_ips}"]
 }
+
+resource "aws_route53_record" "control_plane_nodes" {
+  count = "${length(var.control_plane_ips)}"
+
+  type    = "A"
+  ttl     = "60"
+  zone_id = "${aws_route53_zone.cluster.zone_id}"
+  name    = "control-plane-${count.index}.${var.cluster_domain}"
+  records = ["${var.control_plane_ips[count.index]}"]
+}
+
+resource "aws_route53_record" "compute_nodes" {
+  count = "${length(var.compute_ips)}"
+
+  type    = "A"
+  ttl     = "60"
+  zone_id = "${aws_route53_zone.cluster.zone_id}"
+  name    = "compute-${count.index}.${var.cluster_domain}"
+  records = ["${var.compute_ips[count.index]}"]
+}
diff --git a/upi/vsphere/route53/variables.tf b/upi/vsphere/route53/variables.tf
index 99c8fb6c3d5..d5af77333e4 100644
--- a/upi/vsphere/route53/variables.tf
+++ b/upi/vsphere/route53/variables.tf
@@ -11,6 +11,10 @@ variable "control_plane_ips" {
   type = "list"
 }
 
+variable "compute_ips" {
+  type = "list"
+}
+
 variable "base_domain" {
   description = "The base domain used for public records."
   type        = "string"
diff --git a/upi/vsphere/variables.tf b/upi/vsphere/variables.tf
index 2392f8882c4..c7d6e72d46e 100644
--- a/upi/vsphere/variables.tf
+++ b/upi/vsphere/variables.tf
@@ -124,3 +124,8 @@ variable "compute_instance_count" {
 variable "compute_ignition" {
   type = "string"
 }
+
+variable "compute_ips" {
+  type    = "list"
+  default = []
+}

From 4e08b3be69ac0f393fbe0e1e2272d1ac04034efc Mon Sep 17 00:00:00 2001
From: staebler <staebler@redhat.com>
Date: Thu, 28 Mar 2019 13:23:57 -0400
Subject: [PATCH 2/4] upi/vsphere: put pull secret directly in vm ignition
 config

The bootstrap machine pulls ignition config from a publicly-accessible
url. The pull secret contains sensitive information that should not
be publicly accessible. These changes strip the pull secret out of the
ignition config posted publicly and instead includes it in the ignition
config set directly in the vm.
---
 upi/vsphere/README.md                |  5 ++++-
 upi/vsphere/machine/main.tf          | 24 ++++++++++++++++++++++++
 upi/vsphere/machine/variables.tf     |  5 +++++
 upi/vsphere/main.tf                  |  1 +
 upi/vsphere/terraform.tfvars.example |  8 ++++++++
 upi/vsphere/variables.tf             |  4 ++++
 6 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/upi/vsphere/README.md b/upi/vsphere/README.md
index 6bae41b8c4d..733f39fa3c3 100644
--- a/upi/vsphere/README.md
+++ b/upi/vsphere/README.md
@@ -17,9 +17,12 @@ sshKey: YOUR_SSH_KEY
 2. Run `openshift-install create ignition-configs`.
 
 3. Fill out a terraform.tfvars file with the ignition configs generated.
-There is an example terraform.tfvars file in this directory named terraform.tfvars.example. The example file is set up for use with the dev cluster running at vcsa.vmware.devcluster.openshift.com. At a minimum, you need to set values for `cluster_id`, `cluster_domain`, `vsphere_user`, `vsphere_password`, `bootstrap_ignition_url`, `control_plane_ignition`, and `compute_ignition`.
+There is an example terraform.tfvars file in this directory named terraform.tfvars.example. The example file is set up for use with the dev cluster running at vcsa.vmware.devcluster.openshift.com. At a minimum, you need to set values for `cluster_id`, `cluster_domain`, `vsphere_user`, `vsphere_password`, `pull_secret`, `bootstrap_ignition_url`, `control_plane_ignition`, and `compute_ignition`.
 The bootstrap ignition config must be placed in a location that will be accessible by the bootstrap machine. For example, you could store the bootstrap ignition config in a gist.
 Initially, the `bootstrap_complete` variable must be false, the `bootstrap_ip` variable must be an empty string, and the `control_plane_ips variable must be an empty list.
+To secure your pull secret, you should remove the pull from the bootstrap ignition config and pass it as a variable to terraform.
+  a) Create an ignition config without the pull secret via `jq 'del(.storage.files[] | select(.path=="/root/.docker/config.json"))' bootstrap.ign`.
+  b) Extract the pull secret to pass to terraform via `jq '.storage.files[] | select(.path=="/root/.docker/config.json")' bootstrap.ign`.
 
 4. Run `terraform init`.
 
diff --git a/upi/vsphere/machine/main.tf b/upi/vsphere/machine/main.tf
index 7f3394514af..01c85558cce 100644
--- a/upi/vsphere/machine/main.tf
+++ b/upi/vsphere/machine/main.tf
@@ -1,5 +1,25 @@
 locals {
   ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition)}"
+
+  pull_secret_ignition = <<EOF
+{
+  "ignition": {
+    "config": {},
+    "security": {
+      "tls": {}
+    },
+    "timeouts": {},
+    "version": "2.2.0"
+  },
+  "storage": {
+    "files": [
+${var.pull_secret}
+    ]
+  }
+}
+EOF
+
+  pull_secret_ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(local.pull_secret_ignition)}"
 }
 
 data "vsphere_datastore" "datastore" {
@@ -43,6 +63,10 @@ data "ignition_config" "ign" {
     source = "${var.ignition_url != "" ? var.ignition_url : local.ignition_encoded}"
   }
 
+  append {
+    source = "${local.pull_secret_ignition_encoded}"
+  }
+
   files = [
     "${data.ignition_file.hostname.*.id[count.index]}",
   ]
diff --git a/upi/vsphere/machine/variables.tf b/upi/vsphere/machine/variables.tf
index 3fdd0d7f43f..057b70cc29b 100644
--- a/upi/vsphere/machine/variables.tf
+++ b/upi/vsphere/machine/variables.tf
@@ -47,3 +47,8 @@ variable "datacenter_id" {
 variable "template" {
   type = "string"
 }
+
+variable "pull_secret" {
+  type    = "string"
+  default = ""
+}
diff --git a/upi/vsphere/main.tf b/upi/vsphere/main.tf
index 282414f34b4..284add4329b 100644
--- a/upi/vsphere/main.tf
+++ b/upi/vsphere/main.tf
@@ -29,6 +29,7 @@ module "bootstrap" {
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  pull_secret      = "${var.pull_secret}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
diff --git a/upi/vsphere/terraform.tfvars.example b/upi/vsphere/terraform.tfvars.example
index c7a88431c4d..b8611f45fc1 100644
--- a/upi/vsphere/terraform.tfvars.example
+++ b/upi/vsphere/terraform.tfvars.example
@@ -47,7 +47,15 @@ vsphere_datastore = "nvme-ds1"
 // The "rhcos-davis-no-ig" template is a rhel7 rchos.
 vm_template = "rhcos-davis-no-ig"
 
+// Pull secret ignition snippet extracted from bootstrap ignition config.
+// jq '.storage.files[] | select(.path=="/root/.docker/config.json")' bootstrap.ign
+pull_secret = <<END_OF_PULL_SECRET
+Copy your pull secret ignition snippet here.
+END_OF_PULL_SECRET
+
 // URL of the bootstrap ignition. This needs to be publicly accessible so that the bootstrap machine can pull the ignition.
+// You should remove the pull secret from the ignition config.
+// jq '.storage.files[] | select(.path=="/root/.docker/config.json")' bootstrap.ign
 bootstrap_ignition_url = "URL_FOR_YOUR_BOOTSTRAP_IGNITION"
 
 // Ignition config for the control plane machines. You should copy the contents of the master.ign generated by the installer.
diff --git a/upi/vsphere/variables.tf b/upi/vsphere/variables.tf
index c7d6e72d46e..c309b29f0b7 100644
--- a/upi/vsphere/variables.tf
+++ b/upi/vsphere/variables.tf
@@ -72,6 +72,10 @@ variable "cluster_domain" {
   description = "The base DNS zone to add the sub zone to."
 }
 
+variable "pull_secret" {
+  type = "string"
+}
+
 /////////
 // Bootstrap machine variables
 /////////

From bca44b52b8875b3a1f492cac8a41fc031ffdcc71 Mon Sep 17 00:00:00 2001
From: staebler <staebler@redhat.com>
Date: Tue, 26 Mar 2019 23:05:53 -0400
Subject: [PATCH 3/4] upi/vsphere: use static ip addresses

Static IP addresses are now working with rhcos.
---
 upi/vsphere/README.md                | 48 +++++++++++------------
 upi/vsphere/machine/main.tf          | 58 ++++++++++++++++++++++++----
 upi/vsphere/machine/variables.tf     | 12 ++++--
 upi/vsphere/main.tf                  | 15 ++++---
 upi/vsphere/route53/main.tf          | 16 +-------
 upi/vsphere/terraform.tfvars.example | 29 ++++++--------
 upi/vsphere/variables.tf             | 41 +++++++-------------
 7 files changed, 116 insertions(+), 103 deletions(-)

diff --git a/upi/vsphere/README.md b/upi/vsphere/README.md
index 733f39fa3c3..f54aa0bf72e 100644
--- a/upi/vsphere/README.md
+++ b/upi/vsphere/README.md
@@ -17,42 +17,38 @@ sshKey: YOUR_SSH_KEY
 2. Run `openshift-install create ignition-configs`.
 
 3. Fill out a terraform.tfvars file with the ignition configs generated.
-There is an example terraform.tfvars file in this directory named terraform.tfvars.example. The example file is set up for use with the dev cluster running at vcsa.vmware.devcluster.openshift.com. At a minimum, you need to set values for `cluster_id`, `cluster_domain`, `vsphere_user`, `vsphere_password`, `pull_secret`, `bootstrap_ignition_url`, `control_plane_ignition`, and `compute_ignition`.
+There is an example terraform.tfvars file in this directory named terraform.tfvars.example. The example file is set up for use with the dev cluster running at vcsa.vmware.devcluster.openshift.com. At a minimum, you need to set values for the following variables.
+* cluster_id
+* cluster_domain
+* vsphere_user
+* vsphere_password
+* bootstrap_ip
+* control_plane_ips
+* compute_ips
+* pull_secret
+* bootstrap_ignition_url
+* control_plane_ignition
+* compute_ignition
 The bootstrap ignition config must be placed in a location that will be accessible by the bootstrap machine. For example, you could store the bootstrap ignition config in a gist.
-Initially, the `bootstrap_complete` variable must be false, the `bootstrap_ip` variable must be an empty string, and the `control_plane_ips variable must be an empty list.
+For the IP addresses, you should have static IP addresses reserved for you.
 To secure your pull secret, you should remove the pull from the bootstrap ignition config and pass it as a variable to terraform.
   a) Create an ignition config without the pull secret via `jq 'del(.storage.files[] | select(.path=="/root/.docker/config.json"))' bootstrap.ign`.
   b) Extract the pull secret to pass to terraform via `jq '.storage.files[] | select(.path=="/root/.docker/config.json")' bootstrap.ign`.
 
 4. Run `terraform init`.
 
-5. Run `terraform apply -auto-approve`.
+5. Ensure that you have you AWS profile set and a region specified. The installation will use create AWS route53 resources for routing to the OpenShift cluster.
 
-6. Find the IP address of the bootstrap machine.
-If you provided an extra user, you can use that user to log into the bootstrap machine via the vSphere web console.
-Alternatively, you could iterate through the IP addresses in the 139.178.89.192/26 block looking for one that has the expected hostname, which is bootstrap-0.{cluster_domain}. For example, `ssh -i ~/.ssh/libra.pem -o StrictHostNameChecking=no -q core@139.178.89.199 hostname`
+6. Run `terraform apply -auto-approve`.
+This will create the OpenShift cluster
 
-7. Update the terraform.tfvars file with the IP address of the bootstrap machine.
+7. Run `openshift-install upi bootstrap-complete`. Wait for the bootstrapping to complete.
 
-8. Run `terraform apply -auto-approve`.
-From this point forward, route53 resources will be managed by terraform. You will need to have your AWS profile set and a region specified.
+8. Run `terraform apply -auto-approve -var 'bootstrap_ip='`.
+This will destroy the bootstrap VM.
 
-9. Find the IP addresses of the control plane machines. See step 6 for examples of how to do this. The expected hostnames are control-plane-{0,1,2}.{cluster_domain}. The control plane machines will change their IP addresses once. You need the final IP addresses. If you happen to use the first set of IP addresses, you can later update the IP addresses in the terraform.tfvars file and re-run terraform.
+9. Run `openshift-install upi finish`. Wait for the cluster install to finish.
 
-10. Update the terraform.tfvars file with the IP addresses of the control plane machines.
+10. Enjoy your new OpenShift cluster.
 
-11. Run `terraform apply -auto-approve`.
-
-12. Run `openshift-install user-provided-infrastructure`. Wait for the bootstrapping to complete.
-You *may* need to log into each of the control plane machines. It would seem that, for some reason, the etcd-member pod does not start until the machine is logged into.
-
-13. Update the terraform.tfvars file to set the `bootstrap_complete` variable to "true".
-
-14. Run `terraform apply -auto-approve`.
-
-15. Run `openshift-install user-provided-infrastructure finish`. Wait for the cluster install to finish.
-Currently, the cluster install does not finish. There is an outstanding issue with the openshift-console operator not installing successfully. The cluster should still be usable save for the console, however.
-
-16. Enjoy your new OpenShift cluster.
-
-17. Run `terraform destroy -auto-approve`.
+11. Run `terraform destroy -auto-approve`.
diff --git a/upi/vsphere/machine/main.tf b/upi/vsphere/machine/main.tf
index 01c85558cce..8c567dd0b1b 100644
--- a/upi/vsphere/machine/main.tf
+++ b/upi/vsphere/machine/main.tf
@@ -1,4 +1,7 @@
 locals {
+  mask = "${element(split("/", var.machine_cidr), 1)}"
+  gw   = "${cidrhost(var.machine_cidr,1)}"
+
   ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition)}"
 
   pull_secret_ignition = <<EOF
@@ -38,7 +41,7 @@ data "vsphere_virtual_machine" "template" {
 }
 
 data "ignition_file" "hostname" {
-  count = "${var.instance_count}"
+  count = "${length(var.ips)}"
 
   filesystem = "root"
   path       = "/etc/hostname"
@@ -49,6 +52,44 @@ data "ignition_file" "hostname" {
   }
 }
 
+data "ignition_file" "static_ip" {
+  count = "${length(var.ips)}"
+
+  filesystem = "root"
+  path       = "/etc/sysconfig/network-scripts/ifcfg-eth0"
+  mode       = "420"
+
+  content {
+    content = <<EOF
+TYPE=Ethernet
+BOOTPROTO=none
+NAME=eth0
+DEVICE=eth0
+ONBOOT=yes
+IPADDR=${var.ips[count.index]}
+PREFIX=${local.mask}
+GATEWAY=${local.gw}
+DNS1=8.8.8.8
+EOF
+  }
+}
+
+data "ignition_systemd_unit" "restart" {
+  count = "${length(var.ips)}"
+
+  name = "restart.service"
+
+  content = <<EOF
+[Unit]
+ConditionFirstBoot=yes
+[Service]
+Type=idle
+ExecStart=/sbin/reboot
+[Install]
+WantedBy=multi-user.target
+EOF
+}
+
 data "ignition_user" "extra_users" {
   count = "${length(var.extra_user_names)}"
 
@@ -57,7 +98,7 @@ data "ignition_user" "extra_users" {
 }
 
 data "ignition_config" "ign" {
-  count = "${var.instance_count}"
+  count = "${length(var.ips)}"
 
   append {
     source = "${var.ignition_url != "" ? var.ignition_url : local.ignition_encoded}"
@@ -67,15 +108,20 @@ data "ignition_config" "ign" {
     source = "${local.pull_secret_ignition_encoded}"
   }
 
+  systemd = [
+    "${data.ignition_systemd_unit.restart.*.id[count.index]}",
+  ]
+
   files = [
     "${data.ignition_file.hostname.*.id[count.index]}",
+    "${data.ignition_file.static_ip.*.id[count.index]}",
   ]
 
   users = ["${data.ignition_user.extra_users.*.id}"]
 }
 
 resource "vsphere_virtual_machine" "vm" {
-  count = "${var.instance_count}"
+  count = "${length(var.ips)}"
 
   name             = "${var.name}-${count.index}"
   resource_pool_id = "${var.resource_pool_id}"
@@ -84,9 +130,6 @@ resource "vsphere_virtual_machine" "vm" {
   memory           = "8192"
   guest_id         = "other26xLinux64Guest"
 
-  wait_for_guest_net_timeout  = 0
-  wait_for_guest_net_routable = false
-
   network_interface {
     network_id = "${data.vsphere_network.network.id}"
   }
@@ -103,7 +146,8 @@ resource "vsphere_virtual_machine" "vm" {
 
   vapp {
     properties {
-      "guestinfo.coreos.config.data" = "${data.ignition_config.ign.*.rendered[count.index]}"
+      "guestinfo.ignition.config.data"          = "${base64encode(data.ignition_config.ign.*.rendered[count.index])}"
+      "guestinfo.ignition.config.data.encoding" = "base64"
     }
   }
 }
diff --git a/upi/vsphere/machine/variables.tf b/upi/vsphere/machine/variables.tf
index 057b70cc29b..0dd998691a2 100644
--- a/upi/vsphere/machine/variables.tf
+++ b/upi/vsphere/machine/variables.tf
@@ -2,10 +2,6 @@ variable "name" {
   type = "string"
 }
 
-variable "instance_count" {
-  type = "string"
-}
-
 variable "ignition" {
   type    = "string"
   default = ""
@@ -52,3 +48,11 @@ variable "pull_secret" {
   type    = "string"
   default = ""
 }
+
+variable "machine_cidr" {
+  type = "string"
+}
+
+variable "ips" {
+  type = "list"
+}
diff --git a/upi/vsphere/main.tf b/upi/vsphere/main.tf
index 284add4329b..0c9fd4398dd 100644
--- a/upi/vsphere/main.tf
+++ b/upi/vsphere/main.tf
@@ -21,7 +21,6 @@ module "bootstrap" {
   source = "./machine"
 
   name             = "bootstrap"
-  instance_count   = "${var.bootstrap_complete ? 0 : 1}"
   ignition_url     = "${var.bootstrap_ignition_url}"
   resource_pool_id = "${module.resource_pool.pool_id}"
   datastore        = "${var.vsphere_datastore}"
@@ -30,6 +29,8 @@ module "bootstrap" {
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
   pull_secret      = "${var.pull_secret}"
+  ips              = ["${compact(list(var.bootstrap_ip))}"]
+  machine_cidr     = "${var.machine_cidr}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -39,7 +40,6 @@ module "control_plane" {
   source = "./machine"
 
   name             = "control-plane"
-  instance_count   = "${var.control_plane_instance_count}"
   ignition         = "${var.control_plane_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
   datastore        = "${var.vsphere_datastore}"
@@ -47,6 +47,8 @@ module "control_plane" {
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  ips              = "${var.control_plane_ips}"
+  machine_cidr     = "${var.machine_cidr}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -56,7 +58,6 @@ module "compute" {
   source = "./machine"
 
   name             = "compute"
-  instance_count   = "${var.compute_instance_count}"
   ignition         = "${var.compute_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
   datastore        = "${var.vsphere_datastore}"
@@ -64,6 +65,8 @@ module "compute" {
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  ips              = "${var.compute_ips}"
+  machine_cidr     = "${var.machine_cidr}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -74,7 +77,7 @@ module "dns" {
 
   base_domain       = "${var.base_domain}"
   cluster_domain    = "${var.cluster_domain}"
-  bootstrap_ip      = "${var.bootstrap_complete ? "" : var.bootstrap_ip}"
-  control_plane_ips = "${var.control_plane_ips}"
-  compute_ips       = "${var.compute_ips}"
+  bootstrap_ip      = "${var.bootstrap_ip}"
+  control_plane_ips = ["${var.control_plane_ips}"]
+  compute_ips       = ["${var.compute_ips}"]
 }
diff --git a/upi/vsphere/route53/main.tf b/upi/vsphere/route53/main.tf
index 3a6932ece15..5575a1ad414 100644
--- a/upi/vsphere/route53/main.tf
+++ b/upi/vsphere/route53/main.tf
@@ -1,14 +1,8 @@
-locals {
-  route53_zone_count = "${length(var.control_plane_ips) + length(var.bootstrap_ip) == "0" ? "0" : "1"}"
-}
-
 data "aws_route53_zone" "base" {
   name = "${var.base_domain}"
 }
 
 resource "aws_route53_zone" "cluster" {
-  count = "${local.route53_zone_count}"
-
   name          = "${var.cluster_domain}"
   force_destroy = true
 
@@ -18,8 +12,6 @@ resource "aws_route53_zone" "cluster" {
 }
 
 resource "aws_route53_record" "name_server" {
-  count = "${local.route53_zone_count}"
-
   name    = "${var.cluster_domain}"
   type    = "NS"
   ttl     = "300"
@@ -28,14 +20,12 @@ resource "aws_route53_record" "name_server" {
 }
 
 resource "aws_route53_record" "api" {
-  count = "${local.route53_zone_count}"
-
   type           = "A"
   ttl            = "60"
   zone_id        = "${aws_route53_zone.cluster.zone_id}"
   name           = "api.${var.cluster_domain}"
   set_identifier = "api"
-  records        = "${compact(concat(list(var.bootstrap_ip), var.control_plane_ips))}"
+  records        = ["${compact(concat(list(var.bootstrap_ip), var.control_plane_ips))}"]
 
   weighted_routing_policy {
     weight = 90
@@ -53,8 +43,6 @@ resource "aws_route53_record" "etcd_a_nodes" {
 }
 
 resource "aws_route53_record" "etcd_cluster" {
-  count = "${length(var.control_plane_ips) == "0" ? "0" : "1"}"
-
   type    = "SRV"
   ttl     = "60"
   zone_id = "${aws_route53_zone.cluster.zone_id}"
@@ -63,8 +51,6 @@ resource "aws_route53_record" "etcd_cluster" {
 }
 
 resource "aws_route53_record" "ingress" {
-  count = "${var.compute_instance_count == "0" ? "0" : "1"}"
-
   type    = "A"
   ttl     = "60"
   zone_id = "${aws_route53_zone.cluster.zone_id}"
diff --git a/upi/vsphere/terraform.tfvars.example b/upi/vsphere/terraform.tfvars.example
index b8611f45fc1..d3adb5be85c 100644
--- a/upi/vsphere/terraform.tfvars.example
+++ b/upi/vsphere/terraform.tfvars.example
@@ -1,14 +1,3 @@
-// Set to true once the bootstrapping is complete. The bootstrap machine will be destroyed if this variable is set to "true".
-//bootstrap_complete = true
-
-// The IP address of the bootstrap node.
-// If using the dev vSphere cluster, this IP will be in the 139.178.89.192/26 block.
-//bootstrap_ip = "139.178.89.xxx"
-
-// The IP addresses of the control plan nodes.
-// If using the dev vSphere cluster, this IP will be in the 139.178.89.192/26 block.
-//control_plane_ips = ["139.178.89.xxx","139.178.89.xxx","139.178.89.xxx"]
-
 // ID identifying the cluster to create. Use your username so that resources created can be tracked back to you.
 cluster_id = "example-cluster"
 
@@ -40,12 +29,18 @@ vsphere_datacenter = "dc1"
 // Name of the vSphere data store to use for the VMs. The dev cluster uses "nvme-ds1".
 vsphere_datastore = "nvme-ds1"
 
-// Name of the VM template to clone to create VMs for the cluster. The dev cluster has templates named "rhcos-latest" and "rhcos-davis-no-ig".
-// The "rhcos-latest" template is a recent version of rhcos. There is an issue running the journald gateway on the bootstrap machine with the rhel8 rchos.
-// If you want to use the latest rhcos, you should remove the systemd-journal-gatewayd systemd units from the bootstrap ignition config for the
-// time being.
-// The "rhcos-davis-no-ig" template is a rhel7 rchos.
-vm_template = "rhcos-davis-no-ig"
+// Name of the VM template to clone to create VMs for the cluster. The dev cluster has a template named "rhcos-latest".
+vm_template = "rhcos-latest"
+
+// IP for the bootstrap machine
+// Leave this blank to destroy the bootstrap machine.
+bootstrap_ip = "139.178.89.xxx"
+
+// IP addresses for the control plane machines.
+control_plane_ips = ["139.178.89.xxx","139.178.89.xxx","139.178.89.xxx"]
+
+// IP addresses for the compute machines.
+compute_ips = ["139.178.89.xxx","139.178.89.xxx","139.178.89.xxx"]
 
 // Pull secret ignition snippet extracted from bootstrap ignition config.
 // jq '.storage.files[] | select(.path=="/root/.docker/config.json")' bootstrap.ign
diff --git a/upi/vsphere/variables.tf b/upi/vsphere/variables.tf
index c309b29f0b7..3ec7372a285 100644
--- a/upi/vsphere/variables.tf
+++ b/upi/vsphere/variables.tf
@@ -73,6 +73,11 @@ variable "cluster_domain" {
 }
 
 variable "pull_secret" {
+  type    = "string"
+  default = ""
+}
+
+variable "machine_cidr" {
   type = "string"
 }
 
@@ -80,56 +85,36 @@ variable "pull_secret" {
 // Bootstrap machine variables
 /////////
 
-variable "bootstrap_ignition_url" {
+variable "bootstrap_ip" {
   type = "string"
 }
 
-variable "bootstrap_complete" {
-  type    = "string"
-  default = "false"
-}
-
-variable "bootstrap_ip" {
-  type        = "string"
-  description = "The IP address in the machine_cidr to apply to the bootstrap."
-  default     = ""
+variable "bootstrap_ignition_url" {
+  type = "string"
 }
 
 ///////////
 // Control Plane machine variables
 ///////////
 
-variable "control_plane_instance_count" {
-  type        = "string"
-  description = "The number of control plane instances to deploy."
-  default     = 3
+variable "control_plane_ips" {
+  type = "list"
 }
 
 variable "control_plane_ignition" {
   type = "string"
 }
 
-variable "control_plane_ips" {
-  type        = "list"
-  description = "The IP addresses in the machine_cidr to apply to the control plane machines."
-  default     = []
-}
-
 //////////
 // Compute machine variables
 //////////
 
-variable "compute_instance_count" {
-  type        = "string"
-  description = "The number of compute instances to deploy."
-  default     = 3
+variable "compute_ips" {
+  type    = "list"
+  default = []
 }
 
 variable "compute_ignition" {
   type = "string"
 }
 
-variable "compute_ips" {
-  type    = "list"
-  default = []
-}

From 118ecb31fe62f2982b1dd6b1609214be5daab727 Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Thu, 28 Mar 2019 08:42:44 -0400
Subject: [PATCH 4/4] [WIP] vSphere UPI - Use a folder for virtual machines

- Creates a vm folder
- Virtual Machines are deployed into the folder created
- Updated README.md
---
 upi/vsphere/folder/main.tf       |  5 +++++
 upi/vsphere/folder/output.tf     |  3 +++
 upi/vsphere/folder/variables.tf  | 11 +++++++++++
 upi/vsphere/machine/main.tf      |  1 +
 upi/vsphere/machine/variables.tf |  4 ++++
 upi/vsphere/main.tf              | 11 +++++++++++
 6 files changed, 35 insertions(+)
 create mode 100644 upi/vsphere/folder/main.tf
 create mode 100644 upi/vsphere/folder/output.tf
 create mode 100644 upi/vsphere/folder/variables.tf

diff --git a/upi/vsphere/folder/main.tf b/upi/vsphere/folder/main.tf
new file mode 100644
index 00000000000..6f5605846f1
--- /dev/null
+++ b/upi/vsphere/folder/main.tf
@@ -0,0 +1,5 @@
+resource "vsphere_folder" "folder" {
+  path          = "${var.path}"
+  type          = "vm"
+  datacenter_id = "${var.datacenter_id}"
+}
diff --git a/upi/vsphere/folder/output.tf b/upi/vsphere/folder/output.tf
new file mode 100644
index 00000000000..d20b194905c
--- /dev/null
+++ b/upi/vsphere/folder/output.tf
@@ -0,0 +1,3 @@
+output "path" {
+  value = "${vsphere_folder.folder.path}"
+}
diff --git a/upi/vsphere/folder/variables.tf b/upi/vsphere/folder/variables.tf
new file mode 100644
index 00000000000..60a466edd2a
--- /dev/null
+++ b/upi/vsphere/folder/variables.tf
@@ -0,0 +1,11 @@
+variable "path" {
+  type = "string"
+}
+
+variable "datacenter_id" {
+  type = "string"
+}
+
+variable "vsphere_cluster" {
+  type = "string"
+}
diff --git a/upi/vsphere/machine/main.tf b/upi/vsphere/machine/main.tf
index 8c567dd0b1b..0275da5651e 100644
--- a/upi/vsphere/machine/main.tf
+++ b/upi/vsphere/machine/main.tf
@@ -129,6 +129,7 @@ resource "vsphere_virtual_machine" "vm" {
   num_cpus         = "4"
   memory           = "8192"
   guest_id         = "other26xLinux64Guest"
+  folder           = "${var.folder_id}"
 
   network_interface {
     network_id = "${data.vsphere_network.network.id}"
diff --git a/upi/vsphere/machine/variables.tf b/upi/vsphere/machine/variables.tf
index 0dd998691a2..a3448d6a69b 100644
--- a/upi/vsphere/machine/variables.tf
+++ b/upi/vsphere/machine/variables.tf
@@ -16,6 +16,10 @@ variable "resource_pool_id" {
   type = "string"
 }
 
+variable "folder_id" {
+  type = "string"
+}
+
 variable "datastore" {
   type = "string"
 }
diff --git a/upi/vsphere/main.tf b/upi/vsphere/main.tf
index 0c9fd4398dd..4bb5e803130 100644
--- a/upi/vsphere/main.tf
+++ b/upi/vsphere/main.tf
@@ -17,6 +17,14 @@ module "resource_pool" {
   vsphere_cluster = "${var.vsphere_cluster}"
 }
 
+module "folder" {
+  source = "./folder"
+
+  path            = "${var.cluster_id}"
+  datacenter_id   = "${data.vsphere_datacenter.dc.id}"
+  vsphere_cluster = "${var.vsphere_cluster}"
+}
+
 module "bootstrap" {
   source = "./machine"
 
@@ -24,6 +32,7 @@ module "bootstrap" {
   ignition_url     = "${var.bootstrap_ignition_url}"
   resource_pool_id = "${module.resource_pool.pool_id}"
   datastore        = "${var.vsphere_datastore}"
+  folder_id        = "${module.folder.path}"
   network          = "${var.vm_network}"
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
@@ -42,6 +51,7 @@ module "control_plane" {
   name             = "control-plane"
   ignition         = "${var.control_plane_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
+  folder_id        = "${module.folder.path}"
   datastore        = "${var.vsphere_datastore}"
   network          = "${var.vm_network}"
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
@@ -60,6 +70,7 @@ module "compute" {
   name             = "compute"
   ignition         = "${var.compute_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
+  folder_id        = "${module.folder.path}"
   datastore        = "${var.vsphere_datastore}"
   network          = "${var.vm_network}"
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"