[UPI][vSphere]Pr1492 vmw network

upi/vsphere/README.md

@@ -17,39 +17,38 @@ sshKey: YOUR_SSH_KEY
 2. Run `openshift-install create ignition-configs`.
 
 3. Fill out a terraform.tfvars file with the ignition configs generated.
-There is an example terraform.tfvars file in this directory named terraform.tfvars.example. The example file is set up for use with the dev cluster running at vcsa.vmware.devcluster.openshift.com. At a minimum, you need to set values for `cluster_id`, `cluster_domain`, `vsphere_user`, `vsphere_password`, `bootstrap_ignition_url`, `control_plane_ignition`, and `compute_ignition`.
+There is an example terraform.tfvars file in this directory named terraform.tfvars.example. The example file is set up for use with the dev cluster running at vcsa.vmware.devcluster.openshift.com. At a minimum, you need to set values for the following variables.
+* cluster_id
+* cluster_domain
+* vsphere_user
+* vsphere_password
+* bootstrap_ip
+* control_plane_ips
+* compute_ips
+* pull_secret
+* bootstrap_ignition_url
+* control_plane_ignition
+* compute_ignition
 The bootstrap ignition config must be placed in a location that will be accessible by the bootstrap machine. For example, you could store the bootstrap ignition config in a gist.
-Initially, the `bootstrap_complete` variable must be false, the `bootstrap_ip` variable must be an empty string, and the `control_plane_ips variable must be an empty list.
+For the IP addresses, you should have static IP addresses reserved for you.
+To secure your pull secret, you should remove the pull from the bootstrap ignition config and pass it as a variable to terraform.
+  a) Create an ignition config without the pull secret via `jq 'del(.storage.files[] | select(.path=="/root/.docker/config.json"))' bootstrap.ign`.
+  b) Extract the pull secret to pass to terraform via `jq '.storage.files[] | select(.path=="/root/.docker/config.json")' bootstrap.ign`.
 
 4. Run `terraform init`.
 
-5. Run `terraform apply -auto-approve`.
+5. Ensure that you have you AWS profile set and a region specified. The installation will use create AWS route53 resources for routing to the OpenShift cluster.
 
-6. Find the IP address of the bootstrap machine.
-If you provided an extra user, you can use that user to log into the bootstrap machine via the vSphere web console.
-Alternatively, you could iterate through the IP addresses in the 139.178.89.192/26 block looking for one that has the expected hostname, which is bootstrap-0.{cluster_domain}. For example, `ssh -i ~/.ssh/libra.pem -o StrictHostNameChecking=no -q core@139.178.89.199 hostname`
+6. Run `terraform apply -auto-approve`.
+This will create the OpenShift cluster
 
-7. Update the terraform.tfvars file with the IP address of the bootstrap machine.
+7. Run `openshift-install upi bootstrap-complete`. Wait for the bootstrapping to complete.
 
-8. Run `terraform apply -auto-approve`.
-From this point forward, route53 resources will be managed by terraform. You will need to have your AWS profile set and a region specified.
+8. Run `terraform apply -auto-approve -var 'bootstrap_ip='`.
+This will destroy the bootstrap VM.
 
-9. Find the IP addresses of the control plane machines. See step 6 for examples of how to do this. The expected hostnames are control-plane-{0,1,2}.{cluster_domain}. The control plane machines will change their IP addresses once. You need the final IP addresses. If you happen to use the first set of IP addresses, you can later update the IP addresses in the terraform.tfvars file and re-run terraform.
+9. Run `openshift-install upi finish`. Wait for the cluster install to finish.
 
-10. Update the terraform.tfvars file with the IP addresses of the control plane machines.
+10. Enjoy your new OpenShift cluster.
 
-11. Run `terraform apply -auto-approve`.
-
-12. Run `openshift-install user-provided-infrastructure`. Wait for the bootstrapping to complete.
-You *may* need to log into each of the control plane machines. It would seem that, for some reason, the etcd-member pod does not start until the machine is logged into.
-
-13. Update the terraform.tfvars file to set the `bootstrap_complete` variable to "true".
-
-14. Run `terraform apply -auto-approve`.
-
-15. Run `openshift-install user-provided-infrastructure finish`. Wait for the cluster install to finish.
-Currently, the cluster install does not finish. There is an outstanding issue with the openshift-console operator not installing successfully. The cluster should still be usable save for the console, however.
-
-16. Enjoy your new OpenShift cluster.
-
-17. Run `terraform destroy -auto-approve`.
+11. Run `terraform destroy -auto-approve`.

upi/vsphere/folder/main.tf

@@ -0,0 +1,5 @@
+resource "vsphere_folder" "folder" {
+  path          = "${var.path}"
+  type          = "vm"
+  datacenter_id = "${var.datacenter_id}"
+}

upi/vsphere/folder/output.tf

@@ -0,0 +1,3 @@
+output "path" {
+  value = "${vsphere_folder.folder.path}"
+}

upi/vsphere/folder/variables.tf

@@ -0,0 +1,11 @@
+variable "path" {
+  type = "string"
+}
+
+variable "datacenter_id" {
+  type = "string"
+}
+
+variable "vsphere_cluster" {
+  type = "string"
+}

upi/vsphere/machine/main.tf

@@ -1,5 +1,28 @@
 locals {
+  mask = "${element(split("/", var.machine_cidr), 1)}"
+  gw   = "${cidrhost(var.machine_cidr,1)}"
+
   ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition)}"
+
+  pull_secret_ignition = <<EOF
+{
+  "ignition": {
+    "config": {},
+    "security": {
+      "tls": {}
+    },
+    "timeouts": {},
+    "version": "2.2.0"
+  },
+  "storage": {
+    "files": [
+${var.pull_secret}
+    ]
+  }
+}
+EOF
+
+  pull_secret_ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(local.pull_secret_ignition)}"
 }
 
 data "vsphere_datastore" "datastore" {
@@ -18,7 +41,7 @@ data "vsphere_virtual_machine" "template" {
 }
 
 data "ignition_file" "hostname" {
-  count = "${var.instance_count}"
+  count = "${length(var.ips)}"
 
   filesystem = "root"
   path       = "/etc/hostname"
@@ -29,6 +52,44 @@ data "ignition_file" "hostname" {
   }
 }
 
+data "ignition_file" "static_ip" {
+  count = "${length(var.ips)}"
+
+  filesystem = "root"
+  path       = "/etc/sysconfig/network-scripts/ifcfg-eth0"
+  mode       = "420"
+
+  content {
+    content = <<EOF
+TYPE=Ethernet
+BOOTPROTO=none
+NAME=eth0
+DEVICE=eth0
+ONBOOT=yes
+IPADDR=${var.ips[count.index]}
+PREFIX=${local.mask}
+GATEWAY=${local.gw}
+DNS1=8.8.8.8
+EOF
+  }
+}
+
+data "ignition_systemd_unit" "restart" {
+  count = "${length(var.ips)}"
+
+  name = "restart.service"
+
+  content = <<EOF
+[Unit]
+ConditionFirstBoot=yes
+[Service]
+Type=idle
+ExecStart=/sbin/reboot
+[Install]
+WantedBy=multi-user.target
+EOF
+}
+
 data "ignition_user" "extra_users" {
   count = "${length(var.extra_user_names)}"
 
@@ -37,31 +98,38 @@ data "ignition_user" "extra_users" {
 }
 
 data "ignition_config" "ign" {
-  count = "${var.instance_count}"
+  count = "${length(var.ips)}"
 
   append {
     source = "${var.ignition_url != "" ? var.ignition_url : local.ignition_encoded}"
   }
 
+  append {
+    source = "${local.pull_secret_ignition_encoded}"
+  }
+
+  systemd = [
+    "${data.ignition_systemd_unit.restart.*.id[count.index]}",
+  ]
+
   files = [
     "${data.ignition_file.hostname.*.id[count.index]}",
+    "${data.ignition_file.static_ip.*.id[count.index]}",
   ]
 
   users = ["${data.ignition_user.extra_users.*.id}"]
 }
 
 resource "vsphere_virtual_machine" "vm" {
-  count = "${var.instance_count}"
+  count = "${length(var.ips)}"
 
   name             = "${var.name}-${count.index}"
   resource_pool_id = "${var.resource_pool_id}"
   datastore_id     = "${data.vsphere_datastore.datastore.id}"
   num_cpus         = "4"
   memory           = "8192"
   guest_id         = "other26xLinux64Guest"
-
-  wait_for_guest_net_timeout  = 0
-  wait_for_guest_net_routable = false
+  folder           = "${var.folder_id}"
 
   network_interface {
     network_id = "${data.vsphere_network.network.id}"
@@ -79,7 +147,14 @@ resource "vsphere_virtual_machine" "vm" {
 
   vapp {
     properties {
-      "guestinfo.coreos.config.data" = "${data.ignition_config.ign.*.rendered[count.index]}"
+      "guestinfo.ignition.config.data"          = "${base64encode(data.ignition_config.ign.*.rendered[count.index])}"
+      "guestinfo.ignition.config.data.encoding" = "base64"
     }
   }
+
+  provisioner "local-exec" {
+    when    = "destroy"
+    command = <<EOF
+curl "http://${var.ipam}/api/removeHost.php?apiapp=address&apitoken=${var.ipam_token}&host=${var.name}-${count.index}.${var.cluster_domain}"EOF
+  }
 }

upi/vsphere/machine/variables.tf

@@ -2,10 +2,6 @@ variable "name" {
   type = "string"
 }
 
-variable "instance_count" {
-  type = "string"
-}
-
 variable "ignition" {
   type    = "string"
   default = ""
@@ -20,6 +16,10 @@ variable "resource_pool_id" {
   type = "string"
 }
 
+variable "folder_id" {
+  type = "string"
+}
+
 variable "datastore" {
   type = "string"
 }
@@ -47,3 +47,24 @@ variable "datacenter_id" {
 variable "template" {
   type = "string"
 }
+
+variable "ipam" {
+  type = "string"
+}
+
+variable "ipam_token" {
+  type = "string"
+}
+
+variable "pull_secret" {
+  type    = "string"
+  default = ""
+}
+
+variable "machine_cidr" {
+  type = "string"
+}
+
+variable "ips" {
+  type = "list"
+}

upi/vsphere/main.tf

@@ -9,6 +9,25 @@ data "vsphere_datacenter" "dc" {
   name = "${var.vsphere_datacenter}"
 }
 
+module "network" {
+  source = "./network"
+
+  machine_cidr        = "${var.machine_cidr}"
+  cluster_domain      = "${var.cluster_domain}"
+  control_plane_count = "${var.control_plane_count}"
+  compute_count       = "${var.compute_count}"
+  ipam                = "${var.ipam}"
+  ipam_token          = "${var.ipam_token}"
+}
+
+module "folder" {
+  source = "./folder"
+
+  path            = "${var.cluster_id}"
+  datacenter_id   = "${data.vsphere_datacenter.dc.id}"
+  vsphere_cluster = "${var.vsphere_cluster}"
+}
+
 module "resource_pool" {
   source = "./resource_pool"
 
@@ -21,14 +40,22 @@ module "bootstrap" {
   source = "./machine"
 
   name             = "bootstrap"
-  instance_count   = "${var.bootstrap_complete ? 0 : 1}"
   ignition_url     = "${var.bootstrap_ignition_url}"
   resource_pool_id = "${module.resource_pool.pool_id}"
   datastore        = "${var.vsphere_datastore}"
+  folder_id        = "${module.folder.path}"
   network          = "${var.vm_network}"
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  pull_secret      = "${var.pull_secret}"
+  ipam             = "${var.ipam}"
+  ipam_token       = "${var.ipam_token}"
+
+  ips = ["${compact(list(module.network.bootstrap_ip))}"]
+
+  //ips          = "${module.network.bootstrap_ip}"
+  machine_cidr = "${var.machine_cidr}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -38,14 +65,18 @@ module "control_plane" {
   source = "./machine"
 
   name             = "control-plane"
-  instance_count   = "${var.control_plane_instance_count}"
   ignition         = "${var.control_plane_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
+  folder_id        = "${module.folder.path}"
   datastore        = "${var.vsphere_datastore}"
   network          = "${var.vm_network}"
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  ips              = "${module.network.control_plane_ips}"
+  ipam             = "${var.ipam}"
+  ipam_token       = "${var.ipam_token}"
+  machine_cidr     = "${var.machine_cidr}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -55,14 +86,18 @@ module "compute" {
   source = "./machine"
 
   name             = "compute"
-  instance_count   = "${var.compute_instance_count}"
   ignition         = "${var.compute_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
+  folder_id        = "${module.folder.path}"
   datastore        = "${var.vsphere_datastore}"
   network          = "${var.vm_network}"
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  ips              = "${module.network.compute_ips}"
+  ipam             = "${var.ipam}"
+  ipam_token       = "${var.ipam_token}"
+  machine_cidr     = "${var.machine_cidr}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -73,6 +108,7 @@ module "dns" {
 
   base_domain       = "${var.base_domain}"
   cluster_domain    = "${var.cluster_domain}"
-  bootstrap_ip      = "${var.bootstrap_complete ? "" : var.bootstrap_ip}"
-  control_plane_ips = "${var.control_plane_ips}"
+  bootstrap_ip      = "${module.network.bootstrap_ip}"
+  control_plane_ips = ["${module.network.control_plane_ips}"]
+  compute_ips       = ["${module.network.compute_ips}"]
 }