Merge pull request #36 from gwklok/master

Grab bag of fixes
"lgtm @rlisagor"

Makefile

@@ -28,7 +28,6 @@ include components/statustty/statustty.mk
 include components/diag/diag.mk
 include components/prospector/prospector.mk
 include components/teamster/teamster.mk
-include components/ipxe/ipxe.mk
 include components/waterfront/waterfront.mk
 
 # Write the .buildnum file if the build number changed since the last build.
@@ -52,7 +51,7 @@ vagrant:
 
 .PHONY: packages images test isobuild
 
-packages images test isobuild isobuild-dev installer statustty diag prospector teamster ipxe:
+packages images test isobuild isobuild-dev installer statustty diag prospector teamster:
 	time vagrant ssh -c "cd ~/operos && make $@-novm BUILD_NUM=$(BUILD_NUM)"
 
 .PHONY: packages-novm
@@ -66,11 +65,11 @@ images-novm:
 isobuild: waterfront
 isobuild-dev: waterfront
 
-isobuild-novm: installer-novm statustty-novm diag-novm teamster-novm prospector-novm ipxe-novm
+isobuild-novm: installer-novm statustty-novm diag-novm teamster-novm prospector-novm
 	cp versions iso/installer/airootfs/root/versions
 	sudo ./vagrant-build.sh -V $(ISO_VERSION) -L OPEROS_$(subst .,_,$(ISO_VERSION)) base node worker controller installer
 
-isobuild-dev-novm: installer-novm statustty-novm diag-novm teamster-novm prospector-novm ipxe-novm keys/testkey
+isobuild-dev-novm: installer-novm statustty-novm diag-novm teamster-novm prospector-novm keys/testkey
 	cp versions iso/installer/airootfs/root/versions
 	sudo ./vagrant-build.sh -V $(ISO_VERSION) -L OPEROS_$(subst .,_,$(ISO_VERSION)) -d base node worker controller installer
 

README.md

@@ -12,7 +12,7 @@ cloud-native platform:
 - [Ceph](http://ceph.com/) for distributed storage
 - [Calico](https://www.projectcalico.org/) for software-defined container networking
 - [Prometheus](https://prometheus.io/) for metrics collection
-- [iPXE](http://ipxe.org/)/[NginX](https://www.nginx.com/)/[SYSLINUX](http://www.syslinux.org) for hardware provisioning
+- [isc-dhcpd](https://www.isc.org/downloads/dhcp)/[NginX](https://www.nginx.com/)/[SYSLINUX](http://www.syslinux.org) for hardware provisioning
 - [Arch Linux](https://www.archlinux.org/) as the platform
 
 In addition to the above, Operos includes several original components:

iso/controller/airootfs/etc/systemd/scripts/ceph-mon-init

@@ -1,12 +1,12 @@
 #!/bin/bash -xe
 # Copyright 2018 Pax Automa Systems, Inc.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #    http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,6 +15,33 @@
 
 . /etc/paxautoma/settings
 
+function wait_for_socket {
+	set +e
+	local attempts=$1
+	shift
+
+	local count=$attempts
+	while [ $count -gt 0 ] ; do
+		if [ -S $@  ] ; then
+			break
+		fi
+		count=$[$count-1]
+		sleep 1
+	done
+
+	set -e
+	if [ $count -eq 0 ] ; then
+		return 1
+	fi
+
+    return 0
+}
+
+export ETCDCTL_API=3
+etcd_cmd () {
+	        /usr/bin/etcdctl --endpoints=http://127.0.0.1:4279  "$@"
+}
+
 if [ ! -s /etc/ceph/ceph.conf ] ; then
 
 mkdir -p /etc/ceph
@@ -54,33 +81,41 @@ chown ceph /etc/ceph/ceph.mon.keyring
 
 su -s /bin/bash -c "/usr/bin/ceph-mon --cluster ceph --mkfs -i ${CHOSTNAME} --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring" ceph
 
-export ETCDCTL_API=3
-
 touch /var/lib/ceph/mon/ceph-${CHOSTNAME}/done
 
-
 systemctl enable ceph.target ceph-mon.target
 systemctl enable ceph-mon@${CHOSTNAME}.service
 systemctl start ceph-mon@${CHOSTNAME}.service
 
-/usr/bin/ceph auth get-or-create mgr.${CHOSTNAME} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${CHOSTNAME}/keyring
-systemctl enable ceph-mgr@${CHOSTNAME}.service
-systemctl start ceph-mgr@${CHOSTNAME}.service
 
-/usr/bin/ceph --cluster ceph auth get-or-create client.kube mon 'allow r, allow command "osdblacklist"' osd 'allow rwx pool=kube'
+wait_for_socket 5 /var/run/ceph/ceph-mon.controller.asok
+if [ $? -ne 0 ] ; then
+	echo "FATAL: Ceph mon did not become available after 5 seconds"
+	exit 1
+fi
 
-sleep 3
+/usr/bin/ceph --cluster ceph auth get-or-create client.kube mon 'allow r, allow command "osdblacklist"' osd 'allow rwx pool=kube'
+/usr/bin/ceph --cluster ceph auth caps client.kube mon "profile rbd" osd "profile rbd pool=kube"
+/usr/bin/ceph auth get-or-create mgr.${CHOSTNAME} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${CHOSTNAME}/keyring
 
-etcd_cmd () {
-	        /usr/bin/etcdctl --endpoints=http://127.0.0.1:4279  "$@"
-}
+/usr/bin/ceph config-key set mgr/prometheus/server_addr ${OPEROS_CONTROLLER_IP}
 
+systemctl enable ceph-mgr@${CHOSTNAME}.service
+systemctl start ceph-mgr@${CHOSTNAME}.service
 
 cat /var/lib/ceph/mon/ceph-controller/keyring | etcd_cmd put "cluster/$OPEROS_INSTALL_ID/secret-ceph-mon-keyring"
 cat /etc/ceph/ceph.client.admin.keyring | etcd_cmd put "cluster/$OPEROS_INSTALL_ID/secret-ceph-client-admin-keyring"
 cat /etc/ceph/ceph.conf | etcd_cmd put "cluster/$OPEROS_INSTALL_ID/ceph-config"
 /usr/bin/ceph auth get client.kube | etcd_cmd put "cluster/$OPEROS_INSTALL_ID/secret-ceph-kube-keyring"
 
+wait_for_socket 5 /var/run/ceph/ceph-mgr.controller.asok
+if [ $? -ne 0 ] ; then
+	echo "FATAL: Ceph mgr did not become available after 5 seconds"
+	exit 1
+fi
+
+/usr/bin/ceph mgr module enable prometheus
+
 systemctl disable operos-ceph-mon-init.service
 
 fi

iso/controller/airootfs/etc/systemd/scripts/start-addons

@@ -49,7 +49,7 @@ kubectl create secret generic ceph-admin-key --type="kubernetes.io/rbd" --from-l
 
 ceph --cluster ceph osd pool create kube 128 128
 
-CEPH_KUBE_CLIENT_KEY=$(ceph --cluster ceph auth get cient.kube | grep "key =" | sed -e "s/key = //")
+CEPH_KUBE_CLIENT_KEY=$(ceph --cluster ceph auth get client.kube | grep "key =" | sed -e "s/key = //")
 
 kubectl create secret generic ceph-secret-kube --type="kubernetes.io/rbd" --from-literal=key1=$CEPH_KUBE_CLIENT_KEY --namespace=default
 

iso/installer/airootfs/root/install/104-worker-boot.sh

@@ -1,12 +1,12 @@
 #!/bin/bash -xe
 # Copyright 2018 Pax Automa Systems, Inc.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #    http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,31 +16,21 @@
 echo \> Setting up worker boot infrastructure >&3
 
 mkdir -p /mnt/etc/paxautoma/iso
-cp /mnt/boot/syslinux-worker/ldlinux.c32 /mnt/etc/paxautoma/iso/
-
-cat > /mnt/etc/paxautoma/iso/operos.ipxe <<EOF
-#!ipxe
-:retry_dhcp
-dhcp && isset \${next-server} || goto retry_dhcp
-echo Operos worker booting from http://\${next-server}:5080
-set 210:string http://\${next-server}:5080/
-set 209:string worker.cfg
-chain http://\${next-server}:5080/boot/syslinux-worker/lpxelinux.0
-EOF
-
+cp /mnt/boot/syslinux-worker/lpxelinux.0 /mnt/boot/
+cp /mnt/boot/syslinux-worker/ldlinux.c32 /mnt/boot/
+mkdir -p /mnt/boot/pxelinux.cfg
 
-cat > /mnt/etc/paxautoma/iso/worker.cfg <<EOF
-PATH syslinux-worker/
+cat > /mnt/boot/pxelinux.cfg/default <<EOF
+PATH /syslinux-worker/
 DEFAULT loadconfig
 
 LABEL loadconfig
-  CONFIG boot/syslinux-worker/syslinux.cfg
-  APPEND boot/
+  CONFIG /syslinux-worker/syslinux.cfg
 EOF
 
 # tftp
 cat > /mnt/etc/conf.d/tftpd <<EOF
-TFTPD_ARGS="--verbose --address ${OPEROS_CONTROLLER_IP} -m /tftpboot/mapfile -u ftp --secure /tftpboot"
+TFTPD_ARGS="--verbose --address ${OPEROS_CONTROLLER_IP} -m /etc/tftpd.mapfile -u ftp --secure /boot"
 EOF
 
 # dhcp
@@ -123,16 +113,14 @@ CLUSTER_SN_START=$(network $OPEROS_CONTROLLER_IP ${OPEROS_NODE_MASK//\/})
 
 CLUSTER_NODE_END=$(end_ip $CLUSTER_NODE_START ${OPEROS_NODE_MASK//\/})
 
-mkdir -p /mnt/etc/dhcpd
-
 cat > /mnt/etc/dhcpd.conf <<EOF
 ddns-update-style none;
 deny bootp;     #default
 authoritative;
 
-include "/etc/dhcpd/ipxe-option-space.conf";
-
 subnet ${CLUSTER_SN_START} netmask ${CLUSTER_SN_MASK} {
+    allow bootp;
+    allow booting;
     range ${CLUSTER_NODE_START} ${CLUSTER_NODE_END};
     option subnet-mask ${CLUSTER_SN_MASK};
     option routers ${OPEROS_CONTROLLER_IP};
@@ -141,112 +129,11 @@ subnet ${CLUSTER_SN_START} netmask ${CLUSTER_SN_MASK} {
     option domain-search "${OPEROS_DNS_DOMAIN}";
     option domain-name-servers ${OPEROS_CONTROLLER_IP};
 
-    include "/etc/dhcpd/ipxe-bootp.conf";
+    next-server $OPEROS_CONTROLLER_IP;
+    filename "lpxelinux.0";
 }
 
 use-host-decl-names on;
 EOF
 
-cat > /mnt/etc/dhcpd/ipxe-option-space.conf <<EOF
-# Declare the iPXE/gPXE/Etherboot option space
-option space ipxe;
-option ipxe-encap-opts code 175 = encapsulate ipxe;
-
-# iPXE options, can be set in DHCP response packet
-option ipxe.priority         code   1 = signed integer 8;
-option ipxe.keep-san         code   8 = unsigned integer 8;
-option ipxe.skip-san-boot    code   9 = unsigned integer 8;
-option ipxe.syslogs          code  85 = string;
-option ipxe.cert             code  91 = string;
-option ipxe.privkey          code  92 = string;
-option ipxe.crosscert        code  93 = string;
-option ipxe.no-pxedhcp       code 176 = unsigned integer 8;
-option ipxe.bus-id           code 177 = string;
-option ipxe.bios-drive       code 189 = unsigned integer 8;
-option ipxe.username         code 190 = string;
-option ipxe.password         code 191 = string;
-option ipxe.reverse-username code 192 = string;
-option ipxe.reverse-password code 193 = string;
-option ipxe.version          code 235 = string;
-option iscsi-initiator-iqn   code 203 = string;
-
-# iPXE feature flags, set in DHCP request packet
-option ipxe.pxeext    code 16 = unsigned integer 8;
-option ipxe.iscsi     code 17 = unsigned integer 8;
-option ipxe.aoe       code 18 = unsigned integer 8;
-option ipxe.http      code 19 = unsigned integer 8;
-option ipxe.https     code 20 = unsigned integer 8;
-option ipxe.tftp      code 21 = unsigned integer 8;
-option ipxe.ftp       code 22 = unsigned integer 8;
-option ipxe.dns       code 23 = unsigned integer 8;
-option ipxe.bzimage   code 24 = unsigned integer 8;
-option ipxe.multiboot code 25 = unsigned integer 8;
-option ipxe.slam      code 26 = unsigned integer 8;
-option ipxe.srp       code 27 = unsigned integer 8;
-option ipxe.nbi       code 32 = unsigned integer 8;
-option ipxe.pxe       code 33 = unsigned integer 8;
-option ipxe.elf       code 34 = unsigned integer 8;
-option ipxe.comboot   code 35 = unsigned integer 8;
-option ipxe.efi       code 36 = unsigned integer 8;
-option ipxe.fcoe      code 37 = unsigned integer 8;
-option ipxe.vlan      code 38 = unsigned integer 8;
-option ipxe.menu      code 39 = unsigned integer 8;
-option ipxe.sdi       code 40 = unsigned integer 8;
-option ipxe.nfs       code 41 = unsigned integer 8;
-
-# Other useful general options
-# http://www.ietf.org/assignments/dhcpv6-parameters/dhcpv6-parameters.txt
-option arch code 93 = unsigned integer 16;
-EOF
-
-cat > /mnt/etc/dhcpd/ipxe-bootp.conf <<EOF
-allow bootp;
-allow booting;
-next-server ${OPEROS_CONTROLLER_IP};
-
-# Disable ProxyDHCP, we're in control of the primary DHCP server
-#option ipxe.no-pxedhcp 1;
-
-# Make sure the iPXE we're loading supports what we need,
-# if not load a full-featured version
-if    exists ipxe.http
-  and exists ipxe.menu
-  and ( ( exists ipxe.pxe
-      and exists ipxe.bzimage
-      and exists ipxe.elf
-  ) or (
-      exists ipxe.efi
-  ) ) {
-    filename "http://${OPEROS_CONTROLLER_IP}:5080/operos.ipxe";
-}
-elsif exists user-class and option user-class = "iPXE" {
-    if option arch = 00:06 {
-        filename "ipxe-x86.efi";
-    } elsif option arch = 00:07 {
-        filename "ipxe-x64.efi";
-    } elsif option arch = 00:00 {
-        filename "ipxe.pxe";
-    }
-}
-elsif exists user-class and option user-class = "gPXE" {
-    filename "ipxe.pxe";
-}
-elsif option arch = 00:06 {
-    # EFI 32-bit
-    filename "ipxe-x86.efi";
-}
-elsif option arch = 00:07 {
-    # EFI 64-bit
-    filename "ipxe-x64.efi";
-}
-elsif option arch = 00:00 {
-    # Legacy BIOS x86 mode
-    filename "ipxe.pxe";
-}
-else {
-    # Unsupported client architecture type, so do nothing
-}
-EOF
-
-
 arch-chroot /mnt systemctl enable dhcpd4@${CONTROLLER_PRIVATE_IF}.service