Merge pull request #3657 from pmtk/low-latency/cleanup-kubelet

USHIFT-3770: Add service to cleanup stale kubelet files on boot
openshift · Jul 24, 2024 · a9c820e · a9c820e
2 parents 63d4240 + 159d2b5
commit a9c820e
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 0 deletions.
diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec
@@ -345,6 +345,9 @@ cat assets/optional/multus/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/
 mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release
 install -p -m644 assets/optional/multus/release-multus-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/
 
+# cleanup kubelet
+install -p -m644 packaging/tuned/microshift-cleanup-kubelet.service %{buildroot}%{_unitdir}/microshift-cleanup-kubelet.service
+
 %pre networking
 
 getent group hugetlbfs >/dev/null || groupadd -r hugetlbfs
@@ -409,6 +412,7 @@ fi
 %{_bindir}/microshift-cleanup-data
 %{_bindir}/microshift-sos-report
 %{_unitdir}/microshift.service
+%{_unitdir}/microshift-cleanup-kubelet.service
 %{_sysconfdir}/crio/crio.conf.d/00-crio-crun.conf
 %{_sysconfdir}/crio/crio.conf.d/10-microshift.conf
 %{_datadir}/microshift/spec/config-openapi-spec.json
@@ -478,6 +482,9 @@ fi
 # Use Git command to generate the log and replace the VERSION string
 # LANG=C git log --date="format:%a %b %d %Y" --pretty="tformat:* %cd %an <%ae> VERSION%n- %s%n" packaging/rpm/microshift.spec
 %changelog
+* Thu Jul 18 2024 Patryk Matuszak <pmatusza@redhat.com> 4.17.0
+- Add service to cleanup stale kubelet files on boot
+
 * Mon Jul 08 2024 Pablo Acevedo Montserrat <pacevedo@redhat.com> 4.17.0
 - Add NM configuration file
 

diff --git a/packaging/tuned/microshift-cleanup-kubelet.service b/packaging/tuned/microshift-cleanup-kubelet.service
@@ -0,0 +1,15 @@
+# There are systems with multiple NUMA node that can change reported memory after rebooting
+# causing kubelet's Memory Manager to block startup.
+# To work around this, this unit will delete stale files on boot.
+# Borrowed from: https://github.com/openshift/machine-config-operator/pull/4124
+[Unit]
+Description=Delete Kubelet CPU and Memory managers state on boot
+Before=microshift.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/rm -f /var/lib/kubelet/cpu_manager_state
+ExecStart=/bin/rm -f /var/lib/kubelet/memory_manager_state
+
+[Install]
+WantedBy=multi-user.target
diff --git a/scripts/microshift-cleanup-data.sh b/scripts/microshift-cleanup-data.sh
@@ -114,6 +114,8 @@ function clean_data() {
     if ${FULL_CLEAN} ; then
         echo Removing MicroShift configuration
         rm -rf /var/lib/microshift
+        # Best effort - it contains Pod volumes which sometimes cannot be removed (Device or resource busy)
+        rm -rf /var/lib/kubelet 2>/dev/null || true
     elif ${CERT_CLEAN} ; then
         echo Removing MicroShift certificates
         rm -rf /var/lib/microshift/certs