From 01434e96efae2f0908746cd22b45252af99c162a Mon Sep 17 00:00:00 2001 From: Jakob Meng Date: Wed, 13 Sep 2023 14:56:11 +0200 Subject: [PATCH] [DNM][DRAFT] Changed OKD/FCOS workaround to also support rendezvous host of Agent-based Installer --- ...pender => 30-local-dns-prepender.template} | 1 + .../usr/local/bin/bootstrap-pivot.sh.template | 66 ++++++++++++++----- .../systemd/units/kubelet.service.template | 4 +- .../release-image-pivot.service.template | 4 -- pkg/asset/ignition/bootstrap/common.go | 2 + 5 files changed, 54 insertions(+), 23 deletions(-) rename data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/{30-local-dns-prepender => 30-local-dns-prepender.template} (95%) diff --git a/data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/30-local-dns-prepender b/data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/30-local-dns-prepender.template similarity index 95% rename from data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/30-local-dns-prepender rename to data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/30-local-dns-prepender.template index c20ea1a9d85..85c136fb087 100755 --- a/data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/30-local-dns-prepender +++ b/data/data/bootstrap/baremetal/files/etc/NetworkManager/dispatcher.d/30-local-dns-prepender.template @@ -25,6 +25,7 @@ EOF mkdir -p /etc/systemd/resolved.conf.d echo "[Resolve]" > /etc/systemd/resolved.conf.d/60-kni.conf echo "DNS=$DNS_IP" >> /etc/systemd/resolved.conf.d/60-kni.conf + echo "Domains={{.ClusterDomain}}" >> /etc/systemd/resolved.conf.d/60-kni.conf if systemctl -q is-active systemd-resolved; then >&2 echo "NM resolv-prepender: restarting systemd-resolved" systemctl restart systemd-resolved diff --git a/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template b/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template index 86a42bb9cc2..edb8345024f 100644 --- a/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template @@ -42,25 +42,57 @@ if [ ! -f /opt/openshift/.pivot-done ]; then record_service_stage_start "rebase-to-okd-os-image" {{if .IsFCOS -}} mnt="$(podman image mount "${MACHINE_OS_IMAGE}")" -{{- if .BootstrapInPlace }} - # SNO setup boots into Live ISO which cannot be rebased - # https://github.com/coreos/rpm-ostree/issues/4547 - mkdir /var/mnt/{upper,worker} - mount -t overlay overlay -o "lowerdir=/usr:$mnt/usr" /usr - mount -t overlay overlay -o "lowerdir=/etc:$mnt/etc,upperdir=/var/mnt/upper,workdir=/var/mnt/worker" /etc - systemctl daemon-reload - # Workaround for SELinux denials when launching crio.service from overlayfs - setenforce Permissive + # The bootstrap host during SNO installation and the rendezvous host of Agent-based Installer both boot into a Live + # ISO which cannot be rebased. Until rpm-ostree supports this live rebase [0], the following workaround will mount the + # proper OKD/FCOS Machine OS image over the existing mount at /usr and copy new config files to /etc. + # [0] https://github.com/coreos/rpm-ostree/issues/4547 + if grep -q coreos.liveiso= /proc/cmdline; then + mount -t tmpfs -o size=50% none /var/mnt/ + rsync -aHAXx "$mnt/" /var/mnt/ + mount -t overlay overlay -o lowerdir=/usr:/var/mnt/usr /usr + rsync -rlt --ignore-existing /var/mnt/etc/ /etc/ - systemctl start crio.service - # No reboot necessary because SNO setup will reboot system -{{ else }} - pushd "${mnt}/bootstrap" - # shellcheck disable=SC1091 - . ./pre-pivot.sh - popd -{{ end -}} + # Agent-based Installer will launch a ephemeral control plane at the rendezvous host which will create and publish + # Ignition configs for the other master nodes. These Ignition configs must match what the in-cluster control plane + # would generate else machine config operator will fail [0]. Because the rendezvous host is booted with a FCOS Live + # ISO without any OKD/FCOS related changes, we have to copy the manifests from OKD Machine OS manually to the + # bootstrap manifests folder of the rendezvous host. + # [0] https://access.redhat.com/solutions/4970731 + mkdir -p /var/opt/openshift/manifests + cp -av /var/mnt/manifests/*.* /var/opt/openshift/manifests/ + + # Load new systemd unit files and configuration such as crio.service after mounting the content of OKD/FCOS Machine + # OS over /usr and copying new files to /etc + systemctl daemon-reload + + # CoreDNS fails to listen to 127.0.0.53:53 when Agent-based Installer boots its the rendezvous host with a Fedora + # CoreOS bootimage because by default FCOS' systemd-resolved already listens to this port. OKD/FCOS disables + # resolved's stub listener [0] but the resolved must be restarted for this setting to take effect. + # [0] https://github.com/openshift/okd-machine-os/blob/master/overlay.d/99okd/etc/systemd/resolved.conf.d/okd-no-dns-stub.conf + systemctl restart systemd-resolved.service + + # Workaround for SELinux denials when launching crio.service from overlayfs + setenforce Permissive + + # HACK TODO + # [0] https://github.com/openshift/okd-machine-os/blob/master/overlay.d/99okd/usr/lib/systemd/system-preset/00-okd.preset + #systemctl start \ + # gcp-hostnames.service \ + # fix-resolv-conf-dot.service \ + # fix-resolv-conf-coredns.service \ + # qemu-guest-agent.service \ + # okd-selinux.service || true + + systemctl start crio.service + # No reboot necessary because setup will reboot the system automatically + else + pushd "${mnt}/bootstrap" + # shellcheck disable=SC1091 + . ./pre-pivot.sh + popd + fi + record_service_stage_success {{else if .IsSCOS -}} chmod 0644 /etc/containers/registries.conf rpm-ostree rebase --experimental "ostree-unverified-registry:${MACHINE_OS_IMAGE}" diff --git a/data/data/bootstrap/systemd/units/kubelet.service.template b/data/data/bootstrap/systemd/units/kubelet.service.template index 092d4c8e6e1..a19c998c601 100644 --- a/data/data/bootstrap/systemd/units/kubelet.service.template +++ b/data/data/bootstrap/systemd/units/kubelet.service.template @@ -1,7 +1,7 @@ [Unit] Description=Kubernetes Kubelet -Wants=rpc-statd.service crio.service release-image.service -After=crio.service release-image.service +Wants=rpc-statd.service crio.service release-image.service{{if .IsOKD}} release-image-pivot.service{{end}} +After=crio.service release-image.service{{if .IsOKD}} release-image-pivot.service{{end}} [Service] Type=notify diff --git a/data/data/bootstrap/systemd/units/release-image-pivot.service.template b/data/data/bootstrap/systemd/units/release-image-pivot.service.template index e278091cc50..fd3763f44c2 100644 --- a/data/data/bootstrap/systemd/units/release-image-pivot.service.template +++ b/data/data/bootstrap/systemd/units/release-image-pivot.service.template @@ -3,11 +3,7 @@ Description=Pivot bootstrap to the OpenShift Release Image Wants=release-image.service After=release-image.service -{{- if .BootstrapInPlace }} Before=bootkube.service kubelet.service -{{ else }} -Before=bootkube.service -{{ end -}} [Service] Type=oneshot diff --git a/pkg/asset/ignition/bootstrap/common.go b/pkg/asset/ignition/bootstrap/common.go index de21c4ad0ea..1d876a05cce 100644 --- a/pkg/asset/ignition/bootstrap/common.go +++ b/pkg/asset/ignition/bootstrap/common.go @@ -87,6 +87,7 @@ type bootstrapTemplateData struct { APIServerURL string APIIntServerURL string FeatureSet configv1.FeatureSet + ClusterDomain string } // platformTemplateData is the data to use to replace values in bootstrap @@ -338,6 +339,7 @@ func (a *Common) getTemplateData(dependencies asset.Parents, bootstrapInPlace bo APIServerURL: apiURL, APIIntServerURL: apiIntURL, FeatureSet: installConfig.Config.FeatureSet, + ClusterDomain: installConfig.Config.ClusterDomain(), } }