Merge pull request openshift#3230 from squeed/libvirt-master

steps/masters: add libvirt support

Documentation/dev/libvirt-howto.md

@@ -21,15 +21,23 @@ wget https://beta.release.core-os.net/amd64-usr/current/coreos_production_qemu_i
 bunzip2 coreos_production_qemu_image.img.bz2
 ```
 
-Now, copy `examples/tectonic.libvirt.yaml` and customize it. You're ready to begin! The workflow is the same, but only the `install assets` and `install bootstrap` steps are supported.
+Now, copy `examples/tectonic.libvirt.yaml` and customize it. You're ready to begin! The workflow is the same:
 
+```
+tectonic init --config=<path-to-config>
+tectonic install --dir=<clustername>
+```
 
-## Differences between libvirt and aws:
+The cluster should be up and running in about 10-20 minutes, depending on how quickly the container images are downloaded.
 
-1. We use the Libvirt DNS server. So, if you want to resolve those names on your host, you'll need to configure NetworkManager's dns overlay mode (dnsmasq mode)
-1. There isn't a load balancer. We need to manually remap port 6443 to 443
-1. We may not support changing the number of workers.
 
-## Remaining tasks
-1. Provision the masters and update the DNS names
-1. Provision the workers and update the ingress names
+## Differences between libvirt and aws:
+
+1. We use the Libvirt DNS server. So, if you want to resolve those names on your host, you'll need to configure NetworkManager's dns overlay mode (dnsmasq mode):
+    1. Edit `/etc/NetworkManager/NetworkManager.conf` and set `dns=dnsmasq` in section `main`
+    2. Tell dnsmasq to use your cluster. For me, this is: `echo server=/tt.testing/192.168.124.1 
+     sudo tee /etc/NetworkManager/dnsmasq.d/tectonic.conf`
+    3. restart NetworkManager
+1. There isn't a load balancer. This means:
+    1. We need to manually remap ports that the loadbalancer would
+    2. Only the first server (e.g. master) is actually used. If you want to reach another, you have to manually update the domain name.

config.tf

@@ -72,7 +72,7 @@ variable "tectonic_container_images" {
     tectonic_torcx                       = "quay.io/coreos/tectonic-torcx:v0.2.1"
     kube_addon_operator                  = "quay.io/coreos/kube-addon-operator:beryllium-m2"
     tectonic_alm_operator                = "quay.io/coreos/tectonic-alm-operator:v0.4.0"
-    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator:beryllium-m2"
+    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator:d6b0848118e3b7c78d7d1728ee8846d5c6af2412"
     tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator:beryllium-m2"
     tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator:beryllium-m2"
   }

installer/pkg/workflow/utils.go

@@ -20,8 +20,8 @@ const (
 	assetsStep       = "assets"
 	topologyStep     = "topology"
 	tncDNSStep       = "tnc_dns"
-	bootstrapOn      = "-var=tectonic_aws_bootstrap=true"
-	bootstrapOff     = "-var=tectonic_aws_bootstrap=false"
+	bootstrapOn      = "-var=tectonic_bootstrap=true"
+	bootstrapOff     = "-var=tectonic_bootstrap=false"
 	mastersStep      = "masters"
 	etcdStep         = "etcd"
 	joinWorkersStep  = "joining_workers"

steps/joining_workers/aws/workers.tf

@@ -10,14 +10,14 @@ provider "aws" {
 }
 
 module "container_linux" {
-  source = "../../modules/container_linux"
+  source = "../../../modules/container_linux"
 
   release_channel = "${var.tectonic_container_linux_channel}"
   release_version = "${var.tectonic_container_linux_version}"
 }
 
 module "workers" {
-  source = "../../modules/aws/worker-asg"
+  source = "../../../modules/aws/worker-asg"
 
   autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}"
   cluster_id                   = "${var.tectonic_cluster_id}"

steps/masters/aws/main.tf

@@ -29,7 +29,7 @@ module "masters" {
   container_linux_version      = "${module.container_linux.version}"
   ec2_type                     = "${var.tectonic_aws_master_ec2_type}"
   extra_tags                   = "${var.tectonic_aws_extra_tags}"
-  instance_count               = "${var.tectonic_aws_bootstrap == "true" ? 1 : var.tectonic_master_count}"
+  instance_count               = "${var.tectonic_bootstrap == "true" ? 1 : var.tectonic_master_count}"
   master_iam_role              = "${var.tectonic_aws_master_iam_role_name}"
   master_sg_ids                = "${concat(var.tectonic_aws_master_extra_sg_ids, list(local.sg_id))}"
   private_endpoints            = "${var.tectonic_aws_private_endpoints}"

steps/masters/aws/masters.variables.tf

@@ -1,3 +1,3 @@
-variable "tectonic_aws_bootstrap" {
+variable "tectonic_bootstrap" {
   type = "string"
 }

steps/masters/libvirt/main.tf

@@ -3,7 +3,7 @@ provider "libvirt" {
 }
 
 locals {
-  master_count = 1 # TODO: merge this with the master step
+  master_count = "${var.tectonic_bootstrap == "true" ? 1 : var.tectonic_master_count}"
 }
 
 resource "libvirt_volume" "master" {
@@ -13,20 +13,28 @@ resource "libvirt_volume" "master" {
   base_volume_id = "${local.libvirt_base_volume_id}"
 }
 
-resource "libvirt_ignition" "master" {
-  count = "${local.master_count}"
-
-  name    = "master${count.index}.ign"
+# The first master node should be booted with the bootstrap ignition configuration
+resource "libvirt_ignition" "master_bootstrap" {
+  name    = "master-bootstrap.ign"
   content = "${local.ignition_bootstrap}"
 }
 
+# Ignition for the remaining masters
+resource "libvirt_ignition" "master" {
+  name    = "master.ign"
+  content = "${file("${path.cwd}/${var.tectonic_ignition_master}")}"
+}
+
 resource "libvirt_domain" "master" {
   count = "${local.master_count}"
 
   name = "master${count.index}"
 
-  memory          = "${var.tectonic_libvirt_master_memory}"
-  coreos_ignition = "${element(libvirt_ignition.master.*.id,count.index)}"
+  memory = "${var.tectonic_libvirt_master_memory}"
+
+  # Override ignition for the first (bootstrap) node. It can't be re-ignited,
+  # but that's okay for us
+  coreos_ignition = "${count.index == 0 ? libvirt_ignition.master_bootstrap.id : libvirt_ignition.master.id}"
 
   disk {
     volume_id = "${element(libvirt_volume.master.*.id, count.index)}"

steps/masters/libvirt/masters.variables.tf

@@ -0,0 +1,3 @@
+variable "tectonic_bootstrap" {
+  type = "string"
+}

steps/tnc_dns/aws/main.tf

@@ -10,7 +10,7 @@ provider "aws" {
 }
 
 resource "aws_route53_record" "tectonic_tnc_cname" {
-  count   = "${var.tectonic_aws_bootstrap == "true" ? 1 : 0}"
+  count   = "${var.tectonic_bootstrap == "true" ? 1 : 0}"
   zone_id = "${local.private_zone_id}"
   name    = "${var.tectonic_cluster_name}-tnc.${var.tectonic_base_domain}"
   type    = "CNAME"
@@ -21,7 +21,7 @@ resource "aws_route53_record" "tectonic_tnc_cname" {
 
 resource "aws_route53_record" "tectonic_tnc_a" {
   depends_on = ["aws_route53_record.tectonic_tnc_cname"]
-  count      = "${var.tectonic_aws_bootstrap == "true" ? 0 : 1}"
+  count      = "${var.tectonic_bootstrap == "true" ? 0 : 1}"
   zone_id    = "${local.private_zone_id}"
   name       = "${var.tectonic_cluster_name}-tnc.${var.tectonic_base_domain}"
   type       = "A"

steps/tnc_dns/aws/tnc.variables.tf

@@ -1,3 +1,3 @@
-variable "tectonic_aws_bootstrap" {
+variable "tectonic_bootstrap" {
   type = "string"
 }