[upi][vsphere] applicable static IP changes

upi/vsphere/machine/main.tf

@@ -1,5 +1,7 @@
 locals {
   ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition)}"
+  mask             = "${element(split("/", var.machine_cidr), 1)}"
+  gw               = "${cidrhost(var.machine_cidr,1)}"
 }
 
 data "vsphere_datastore" "datastore" {
@@ -29,6 +31,35 @@ data "ignition_file" "hostname" {
   }
 }
 
+data "ignition_file" "static_ip" {
+  count = "${var.instance_count}"
+
+  filesystem = "root"
+  path       = "/etc/sysconfig/network-scripts/ifcfg-eth0"
+  mode       = "420"
+
+  content {
+    content = <<EOF
+TYPE=Ethernet
+BOOTPROTO=none
+NAME=eth0
+DEVICE=eth0
+ONBOOT=yes
+IPADDR=${var.ips[count.index]}
+PREFIX=${local.mask}
+GATEWAY=${local.gw}
+DNS1=8.8.8.8
+    EOF
+  }
+}
+
+data "ignition_systemd_unit" "restart" {
+  count = "${var.instance_count}"
+
+  name    = "restart.service"
+  content = "[Unit]\nConditionFirstBoot=yes\n\n[Service]\nType=idle\nExecStart=/sbin/reboot\n\n[Install]\nWantedBy=multi-user.target"
+}
+
 data "ignition_user" "extra_users" {
   count = "${length(var.extra_user_names)}"
 
@@ -43,8 +74,13 @@ data "ignition_config" "ign" {
     source = "${var.ignition_url != "" ? var.ignition_url : local.ignition_encoded}"
   }
 
+  systemd = [
+    "${data.ignition_systemd_unit.restart.*.id[count.index]}",
+  ]
+
   files = [
     "${data.ignition_file.hostname.*.id[count.index]}",
+    "${data.ignition_file.static_ip.*.id[count.index]}",
   ]
 
   users = ["${data.ignition_user.extra_users.*.id}"]
@@ -53,7 +89,7 @@ data "ignition_config" "ign" {
 resource "vsphere_virtual_machine" "vm" {
   count = "${var.instance_count}"
 
-  name             = "${var.name}-${count.index}"
+  name             = "${var.cluster_id}-${var.name}-${count.index}"
   resource_pool_id = "${var.resource_pool_id}"
   datastore_id     = "${data.vsphere_datastore.datastore.id}"
   num_cpus         = "4"
@@ -79,7 +115,8 @@ resource "vsphere_virtual_machine" "vm" {
 
   vapp {
     properties {
-      "guestinfo.coreos.config.data" = "${data.ignition_config.ign.*.rendered[count.index]}"
+      "guestinfo.ignition.config.data"          = "${base64encode(data.ignition_config.ign.*.rendered[count.index])}"
+      "guestinfo.ignition.config.data.encoding" = "base64"
     }
   }
 }

upi/vsphere/machine/variables.tf

@@ -2,6 +2,10 @@ variable "name" {
   type = "string"
 }
 
+variable "cluster_id" {
+  type = "string"
+}
+
 variable "instance_count" {
   type = "string"
 }
@@ -28,6 +32,14 @@ variable "network" {
   type = "string"
 }
 
+variable "machine_cidr" {
+  type = "string"
+}
+
+variable "ips" {
+  type = "list"
+}
+
 variable "cluster_domain" {
   type = "string"
 }

upi/vsphere/main.tf

@@ -9,6 +9,16 @@ data "vsphere_datacenter" "dc" {
   name = "${var.vsphere_datacenter}"
 }
 
+module "network" {
+  source = "./network"
+
+  # TODO Pass in and check the base domain and cluster domain to look for A records
+  machine_cidr   = "${var.machine_cidr}"
+  cluster_domain = "${var.cluster_domain}"
+  master_count   = "${var.control_plane_instance_count}"
+  worker_count   = "${var.compute_instance_count}"
+}
+
 module "resource_pool" {
   source = "./resource_pool"
 
@@ -21,6 +31,8 @@ module "bootstrap" {
   source = "./machine"
 
   name             = "bootstrap"
+  machine_cidr     = "${var.machine_cidr}"
+  ips              = "${module.network.bootstrap_ip}"
   instance_count   = "${var.bootstrap_complete ? 0 : 1}"
   ignition_url     = "${var.bootstrap_ignition_url}"
   resource_pool_id = "${module.resource_pool.pool_id}"
@@ -29,6 +41,7 @@ module "bootstrap" {
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  cluster_id       = "${var.cluster_id}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -38,6 +51,8 @@ module "control_plane" {
   source = "./machine"
 
   name             = "control-plane"
+  machine_cidr     = "${var.machine_cidr}"
+  ips              = "${module.network.master_ips}"
   instance_count   = "${var.control_plane_instance_count}"
   ignition         = "${var.control_plane_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
@@ -46,6 +61,7 @@ module "control_plane" {
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  cluster_id       = "${var.cluster_id}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -55,6 +71,8 @@ module "compute" {
   source = "./machine"
 
   name             = "compute"
+  machine_cidr     = "${var.machine_cidr}"
+  ips              = "${module.network.worker_ips}"
   instance_count   = "${var.compute_instance_count}"
   ignition         = "${var.compute_ignition}"
   resource_pool_id = "${module.resource_pool.pool_id}"
@@ -63,6 +81,7 @@ module "compute" {
   datacenter_id    = "${data.vsphere_datacenter.dc.id}"
   template         = "${var.vm_template}"
   cluster_domain   = "${var.cluster_domain}"
+  cluster_id       = "${var.cluster_id}"
 
   extra_user_names           = ["${var.extra_user_names}"]
   extra_user_password_hashes = ["${var.extra_user_password_hashes}"]
@@ -73,6 +92,6 @@ module "dns" {
 
   base_domain       = "${var.base_domain}"
   cluster_domain    = "${var.cluster_domain}"
-  bootstrap_ip      = "${var.bootstrap_complete ? "" : var.bootstrap_ip}"
-  control_plane_ips = "${var.control_plane_ips}"
+  bootstrap_ip      = "${module.network.bootstrap_ip}"
+  control_plane_ips = "${module.network.master_ips}"
 }

upi/vsphere/network/cidr_to_ip.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+# cidr_to_ip - 
+#  https://www.terraform.io/docs/providers/external/data_source.html
+#  Based on info from here: https://gist.github.com/irvingpop/968464132ded25a206ced835d50afa6b
+#  This script takes the CIDR address and cycles through looking for the first available address
+#  echo '{"cidr": "139.178.89.192/26", "master_count": "3", "worker_count": "3", "cluster_domain": "dphillip.devcluster.openshift.com"}' | ./cidr_to_ip.sh
+function error_exit() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+function check_deps() {
+  test -f $(which jq) || error_exit "jq command not detected in path, please install it"
+  test -f $(which ipcalc) || error_exit "ipcalc command not detected in path, please install it"
+
+}
+
+function parse_input() {
+  # jq reads from stdin so we don't have to set up any inputs, but let's validate the outputs
+  eval "$(jq -r '@sh "export CIDR=\(.cidr) master_count=\(.master_count)  worker_count=\(.worker_count) cluster_domain=\(.cluster_domain)"')"
+  if [[ -z "${CIDR}" ]]; then export CIDR=none; fi
+  if [[ -z "${master_count}" ]]; then export master_count=none; fi
+  if [[ -z "${worker_count}" ]]; then export worker_count=none; fi
+  if [[ -z "${cluster_domain}" ]]; then export cluster_domain=none; fi
+}
+
+function produce_output() {
+
+  cidr=$CIDR
+
+  # range is bounded by network (-n) & broadcast (-b) addresses.
+  lo=$(ipcalc -n $cidr | cut -f2 -d=)
+  hi=$(ipcalc -b $cidr | cut -f2 -d=)
+
+  read a b c d <<< $(echo $lo | tr . ' ')
+  read e f g h <<< $(echo $hi | tr . ' ')
+  IP_RANGE=$(eval echo {$a..$e}.{$b..$f}.{$c..$g}.{$d..$h})
+
+  bs_count=0
+  m_count=0
+  w_count=0
+
+  # check cluster_domain DNS first
+  DNS_RECORDS=$(dig +short api.${cluster_domain}| sort)
+  for ENTRY in ${DNS_RECORDS} 
+  do
+       ping -c1 -w1 $ENTRY > /dev/null 2>&1
+       ping_rc=$?  
+       if [[ $ping_rc -eq 0 ]] && [[ $bs_count -ne 1 ]]
+       then
+	     # delete the bootstrap DNS
+             bootstrap_ip+=""
+             bs_count=$((bs_count+1))
+       elif [[ $ping_rc -eq 0 ]] && [[ $m_count -ne $master_count ]]
+       then
+             master_ips+="$ENTRY "
+             m_count=$((m_count+1))
+       fi
+  done
+
+  for IPADDR in ${IP_RANGE}
+  do
+
+	if [ $IPADDR != $(ipcalc -n $cidr | cut -f2 -d=) ] && [ $IPADDR != $(ipcalc -b $cidr | cut -f2 -d=) ] 
+        then
+  	  ping -c1 -w1 $IPADDR > /dev/null 2>&1
+  	  ping_rc=$?
+
+	  if [[ $ping_rc -eq 1 ]] && [[ $bs_count -ne 1 ]]
+	  then
+  		bootstrap_ip+="$IPADDR"
+		bs_count=$((bs_count+1))
+	  elif [[ $ping_rc -eq 1 ]] && [[ $m_count -ne $master_count ]]
+	  then
+  		master_ips+="$IPADDR "
+		m_count=$((m_count+1))
+	  elif [[ $ping_rc -eq 1 ]] && [[ $w_count -ne $worker_count ]]
+	  then
+  		worker_ips+="$IPADDR "
+		w_count=$((w_count+1))
+	  elif [[ $bs_count -eq 1 ]] && [[ $m_count -eq $master_count ]] && [[ $w_count -eq $worker_count ]]
+	  then
+  		jq -n \
+    		--arg bootstrap_ip "$bootstrap_ip" \
+    		--arg master_ips "$master_ips" \
+    		--arg worker_ips "$worker_ips" \
+    		'{"bootstrap_ip":$bootstrap_ip,"master_ips":$master_ips,"worker_ips":$worker_ips}'
+	        exit 0
+	   fi
+      	 fi
+  done
+}
+
+# main()
+check_deps
+parse_input
+produce_output

upi/vsphere/network/main.tf

@@ -0,0 +1,10 @@
+data "external" "ping" {
+  program = ["bash", "${path.root}/network/cidr_to_ip.sh"]
+
+  query = {
+    cidr           = "${var.machine_cidr}"
+    master_count   = "${var.master_count}"
+    worker_count   = "${var.worker_count}"
+    cluster_domain = "${var.cluster_domain}"
+  }
+}

upi/vsphere/network/output.tf

@@ -0,0 +1,11 @@
+output "master_ips" {
+  value = "${split(" ", trimspace(data.external.ping.result.master_ips))}"
+}
+
+output "worker_ips" {
+  value = "${split(" ", trimspace(data.external.ping.result.worker_ips))}"
+}
+
+output "bootstrap_ip" {
+  value = "${split(" ", trimspace(data.external.ping.result.bootstrap_ip))}"
+}

upi/vsphere/network/variables.tf

@@ -0,0 +1,19 @@
+variable "machine_cidr" {
+  type        = "string"
+  description = "This is the public network netmask."
+}
+
+variable "cluster_domain" {
+  type        = "string"
+  description = "This is the cluster domain where the API record is created"
+}
+
+variable "master_count" {
+  type        = "string"
+  description = "The number of master IP addresses to obtain from the machine_cidr."
+}
+
+variable "worker_count" {
+  type        = "string"
+  description = "The number of worker IP addresses to obtain from the machine_cidr."
+}

upi/vsphere/route53/main.tf

@@ -35,7 +35,7 @@ resource "aws_route53_record" "api" {
   zone_id        = "${aws_route53_zone.cluster.zone_id}"
   name           = "api.${var.cluster_domain}"
   set_identifier = "api"
-  records        = "${compact(concat(list(var.bootstrap_ip), var.control_plane_ips))}"
+  records        = ["${compact(concat(var.bootstrap_ip, var.control_plane_ips))}"]
 
   weighted_routing_policy {
     weight = 90

upi/vsphere/route53/variables.tf

@@ -4,7 +4,7 @@ variable "cluster_domain" {
 }
 
 variable "bootstrap_ip" {
-  type = "string"
+  type = "list"
 }
 
 variable "control_plane_ips" {

upi/vsphere/variables.tf

@@ -85,6 +85,11 @@ variable "bootstrap_complete" {
   default = "false"
 }
 
+variable "machine_cidr" {
+  type        = "string"
+  description = "The IP space to provision with."
+}
+
 variable "bootstrap_ip" {
   type        = "string"
   description = "The IP address in the machine_cidr to apply to the bootstrap."