Render osImageURL and implement earlier pivot

Have the MCC take `osImageURL` as provided by the cluster update/release payload
and generate a `00-{master,worker}-osimageurl` MC from it, which ensures
the MCD will update the node to it.

However, we need special handling for the *initial* case where we boot
into a target config, but we may be using an old OS image.

Change the MCC to write the target osImageURL from the MC it uses for
bootstrapping to `/etc/rhcos-initial-pivot-target`.  This will then be
handled by the `rhcos-initial-pivot.service` systemd unit.

Closes: openshift#183

docs/MachineConfigServer.md

@@ -36,6 +36,10 @@ It performs the following extra actions on the Ignition config defined in the Ma
 
    The new machines that come up, will need a KubeConfig file which will be added as an Ignition file. 
 
+* *Early pivot* - `/etc/rhcos-initial-pivot-target`
+
+    While the Ignition content configures the node, it may actually be booted into an older OS image than is specified by the release payload managed by the [Cluster Version Operator](https://github.com/openshift/cluster-version-operator/).  The MCS writes out the `osImageURL` to this file, and the system will (if necessary) performs an "early pivot" before the node has actually joined the cluster.  This then ensures that when the MachineConfigDaemon starts, it will validate the `currentConfig` (including files written by Ignition and the `osImageURL`).
+
 ### Running MachineConfigServer
 
 It is recommended that the MachineConfigServer is run as a DaemonSet on all `master` machines with the pods running in host network. So machines can access the Ignition endpoint through load balancer setup for control plane.

pkg/controller/template/render.go

@@ -91,21 +91,16 @@ func generateMachineConfigs(config *RenderConfig, templateDir string) ([]*mcfgv1
 	return cfgs, nil
 }
 
+// GenerateMachineConfigsForRole is part of generateMachineConfigs; it operates
+// on a specific role which has a set of builtin templates.
 func GenerateMachineConfigsForRole(config *RenderConfig, role string, path string) ([]*mcfgv1.MachineConfig, error) {
+	cfgs := []*mcfgv1.MachineConfig{}
+
+	// Add our built-in templates
 	infos, err := ioutil.ReadDir(path)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read dir %q: %v", path, err)
 	}
-	// for each role a machine config is created containing the sshauthorized keys to allow for ssh access
-	// ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key
-	var tempIgnConfig ignv2_2types.Config
-	tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}}
-	tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser)
-	sshConfigName := "00-" + role + "-ssh"
-	sshMachineConfigForRole := MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig)
-
-	cfgs := []*mcfgv1.MachineConfig{}
-	cfgs = append(cfgs, sshMachineConfigForRole)
 
 	for _, info := range infos {
 		if !info.IsDir() {
@@ -121,9 +116,57 @@ func GenerateMachineConfigsForRole(config *RenderConfig, role string, path strin
 		cfgs = append(cfgs, nameConfig)
 	}
 
+	// And derived configs
+	derivedCfgs, err := generateDerivedMachineConfigs(config, role)
+	if err != nil {
+		return nil, err
+	}
+	cfgs = append(cfgs, derivedCfgs...)
+
+	return cfgs, nil
+}
+
+// machineConfigForOSImageURL generates a MC fragment that just includes the target OSImageURL.
+func machineConfigForOSImageURL(role string, url string) *mcfgv1.MachineConfig {
+	labels := map[string]string{
+		machineConfigRoleLabelKey: role,
+	}
+	return &mcfgv1.MachineConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels: labels,
+			Name:   "00-" + role + "-osimageurl",
+		},
+		Spec: mcfgv1.MachineConfigSpec{
+			OSImageURL: url,
+		},
+	}
+}
+
+// generateDerivedMachineConfigs is part of generateMachineConfigsForRole. It
+// takes care of generating MachineConfig objects which are derived from other
+// components of the cluster configuration. Currently, that's:
+//
+//  - SSH keys from the install configuration
+//  - OSImageURL from the machine-config-osimageurl configmap (which comes from the CVO)
+func generateDerivedMachineConfigs(config *RenderConfig, role string) ([]*mcfgv1.MachineConfig, error) {
+	cfgs := []*mcfgv1.MachineConfig{}
+
+	// for each role a machine config is created containing the sshauthorized keys to allow for ssh access
+	// ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key
+	var tempIgnConfig ignv2_2types.Config
+	tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}}
+	tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser)
+	sshConfigName := "00-" + role + "-ssh"
+	cfgs = append(cfgs, MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig))
+
+	if config.OSImageURL != "" {
+		cfgs = append(cfgs, machineConfigForOSImageURL(role, config.OSImageURL))
+	}
+
 	return cfgs, nil
 }
 
+// generateMachineConfigForName is part of the implementation of generateMachineConfigsForRole
 func generateMachineConfigForName(config *RenderConfig, role, name, path string) (*mcfgv1.MachineConfig, error) {
 	platformDirs := []string{}
 	for _, dir := range []string{"_base", config.Platform} {

pkg/controller/template/test_data/templates/master/00-master/aws/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/master/00-master/libvirt/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/master/00-master/none/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/master/00-master/openstack/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/worker/00-worker/aws/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/worker/00-worker/libvirt/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/worker/00-worker/none/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/controller/template/test_data/templates/worker/00-worker/openstack/units/rhcos-initial-pivot.service

@@ -0,0 +1,19 @@
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target
+enabled: true
+name: rhcos-initial-pivot.service

pkg/server/bootstrap_server.go

@@ -96,7 +96,7 @@ func (bsc *bootstrapServer) GetConfig(cr poolRequest) (*ignv2_2types.Config, err
 		return nil, fmt.Errorf("server: could not unmarshal file %s, err: %v", fileName, err)
 	}
 
-	appenders := getAppenders(cr, currConf, bsc.kubeconfigFunc)
+	appenders := getAppenders(cr, currConf, bsc.kubeconfigFunc, "")
 	for _, a := range appenders {
 		if err := a(&mc.Spec.Config); err != nil {
 			return nil, err

pkg/server/cluster_server.go

@@ -70,7 +70,7 @@ func (cs *clusterServer) GetConfig(cr poolRequest) (*ignv2_2types.Config, error)
 		return nil, fmt.Errorf("could not fetch config %s, err: %v", currConf, err)
 	}
 
-	appenders := getAppenders(cr, currConf, cs.kubeconfigFunc)
+	appenders := getAppenders(cr, currConf, cs.kubeconfigFunc, mc.Spec.OSImageURL)
 	for _, a := range appenders {
 		if err := a(&mc.Spec.Config); err != nil {
 			return nil, err

pkg/server/server.go

@@ -16,6 +16,9 @@ const (
 	// of the KubeConfig file on the machine.
 	defaultMachineKubeConfPath = "/etc/kubernetes/kubeconfig"
 
+	// rhcosInitialPivotPath is processed by rhcos-initial-pivot.service
+	rhcosInitialPivotPath = "/etc/rhcos-initial-pivot-target"
+
 	// defaultFileSystem defines the default file system to be
 	// used for writing the ignition files created by the
 	// server.
@@ -34,10 +37,11 @@ type Server interface {
 	GetConfig(poolRequest) (*ignv2_2types.Config, error)
 }
 
-func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc) []appenderFunc {
+func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc, osimageurl string) []appenderFunc {
 	appenders := []appenderFunc{
 		// append machine annotations file.
 		func(config *ignv2_2types.Config) error { return appendNodeAnnotations(config, currMachineConfig) },
+		func(config *ignv2_2types.Config) error { return appendInitialPivot(config, osimageurl) },
 		// append kubeconfig.
 		func(config *ignv2_2types.Config) error { return appendKubeConfig(config, f) },
 	}
@@ -53,6 +57,15 @@ func appendKubeConfig(conf *ignv2_2types.Config, f kubeconfigFunc) error {
 	return nil
 }
 
+// Ensures that the node is in the OS we expect; for more information see
+// rhcos-initial-pivot.service in the templates
+func appendInitialPivot(conf *ignv2_2types.Config, osimageurl string) error {
+	if osimageurl != "" {
+		appendFileToIgnition(conf, rhcosInitialPivotPath, osimageurl + "\n")
+	}
+	return nil
+}
+
 func appendNodeAnnotations(conf *ignv2_2types.Config, currConf string) error {
 	anno, err := getNodeAnnotation(currConf)
 	if err != nil {

templates/master/00-master/_base/units/rhcos-initial-pivot.yaml

@@ -0,0 +1,19 @@
+name: "rhcos-initial-pivot.service"
+enabled: true
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target

templates/worker/00-worker/_base/units/rhcos-initial-pivot.yaml

@@ -0,0 +1,19 @@
+name: "rhcos-initial-pivot.service"
+enabled: true
+contents: |
+  [Unit]
+  Description=RHCOS initial pivot
+  # Before we join the cluster
+  Before=kubelet.service
+  # But be sure that we've finished Ignition before we reboot
+  After=ignition-firstboot-complete.service
+  ConditionPathExists=/etc/rhcos-initial-pivot-target
+
+  [Service]
+  Type=simple
+  ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target'
+  Restart=on-failure
+  RestartSec=30
+
+  [Install]
+  WantedBy=multi-user.target

test/e2e/main_test.go

@@ -5,6 +5,10 @@ import (
 	"testing"
 )
 
+const (
+	namespace = "openshift-machine-config-operator"
+)
+
 func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }