feat: Support multiple MOFED DS

Mofed driver precompiled container images
are compiled using a specific Kernel.

As a result, the Mofed Driver DaemonSet should
have the Kernel as part of the Node Selector.

In addition, since there can be Nodes with different
Kernel versions, a DaemonSet for each existing Kernel
in the cluster is created.

In the Migration module, the former DS is deleted
with DeletePropagationOrphan so that MOFED pods will
still exists until manual or auto-upgrade is done.

Signed-off-by: Fred Rolland <frolland@nvidia.com>

controllers/nicclusterpolicy_controller.go

@@ -52,6 +52,7 @@ type NicClusterPolicyReconciler struct {
 	Scheme               *runtime.Scheme
 	ClusterTypeProvider  clustertype.Provider
 	StaticConfigProvider staticconfig.Provider
+	MigrationCh          chan struct{}
 
 	stateManager state.Manager
 }
@@ -87,6 +88,11 @@ type NicClusterPolicyReconciler struct {
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 func (r *NicClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	select {
+	case <-r.MigrationCh:
+	case <-ctx.Done():
+		return ctrl.Result{}, fmt.Errorf("canceled")
+	}
 	reqLogger := log.FromContext(ctx)
 	reqLogger.V(consts.LogLevelInfo).Info("Reconciling NicClusterPolicy")
 
@@ -179,6 +185,10 @@ func (r *NicClusterPolicyReconciler) handleMOFEDWaitLabels(
 	_ = r.Client.List(ctx, pods, client.MatchingLabels{"nvidia.com/ofed-driver": ""})
 	for i := range pods.Items {
 		pod := pods.Items[i]
+		if pod.Spec.NodeName == "" {
+			// In case that Pod is in Pending state
+			continue
+		}
 		labelValue := "true"
 		// We assume that OFED pod contains only one container to simplify the logic.
 		// We can revisit this logic in the future if needed

controllers/suite_test.go

@@ -128,11 +128,15 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	staticConfigProvider := staticconfig.NewProvider(staticconfig.StaticConfig{CniBinDirectory: "/opt/cni/bin"})
 
+	migrationCompletionChan := make(chan struct{})
+	close(migrationCompletionChan)
+
 	err = (&NicClusterPolicyReconciler{
 		Client:               k8sManager.GetClient(),
 		Scheme:               k8sManager.GetScheme(),
 		ClusterTypeProvider:  clusterTypeProvider,
 		StaticConfigProvider: staticConfigProvider,
+		MigrationCh:          migrationCompletionChan,
 	}).SetupWithManager(k8sManager, testSetupLog)
 	Expect(err).ToNot(HaveOccurred())
 

controllers/upgrade_controller.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/NVIDIA/k8s-operator-libs/pkg/upgrade"
@@ -47,6 +48,7 @@ type UpgradeReconciler struct {
 	client.Client
 	Scheme       *runtime.Scheme
 	StateManager upgrade.ClusterUpgradeStateManager
+	MigrationCh  chan struct{}
 }
 
 const plannedRequeueInterval = time.Minute * 2
@@ -64,6 +66,11 @@ const UpgradeStateAnnotation = "nvidia.com/ofed-upgrade-state"
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 func (r *UpgradeReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) {
+	select {
+	case <-r.MigrationCh:
+	case <-ctx.Done():
+		return ctrl.Result{}, fmt.Errorf("canceled")
+	}
 	reqLogger := log.FromContext(ctx)
 	reqLogger.V(consts.LogLevelInfo).Info("Reconciling Upgrade")
 

controllers/upgrade_controller_test.go

@@ -55,9 +55,12 @@ var _ = Describe("Upgrade Controller", func() {
 	})
 	Context("When NicClusterPolicy CR is created", func() {
 		It("Upgrade policy is disabled", func() {
+			migrationCompletionChan := make(chan struct{})
+			close(migrationCompletionChan)
 			upgradeReconciler := &UpgradeReconciler{
-				Client: k8sClient,
-				Scheme: k8sClient.Scheme(),
+				Client:      k8sClient,
+				Scheme:      k8sClient.Scheme(),
+				MigrationCh: migrationCompletionChan,
 			}
 
 			req := ctrl.Request{NamespacedName: types.NamespacedName{Name: consts.NicClusterPolicyResourceName}}
@@ -76,10 +79,13 @@ var _ = Describe("Upgrade Controller", func() {
 				err := k8sClient.Create(goctx.TODO(), node)
 				Expect(err).NotTo(HaveOccurred())
 			}
+			migrationCompletionChan := make(chan struct{})
+			close(migrationCompletionChan)
 
 			upgradeReconciler := &UpgradeReconciler{
-				Client: k8sClient,
-				Scheme: k8sClient.Scheme(),
+				Client:      k8sClient,
+				Scheme:      k8sClient.Scheme(),
+				MigrationCh: migrationCompletionChan,
 			}
 			// Call removeNodeUpgradeStateLabels function
 			err := upgradeReconciler.removeNodeUpgradeStateLabels(goctx.TODO())

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.4
 	github.com/xeipuuv/gojsonschema v1.2.0
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2

go.sum

@@ -196,8 +196,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=

main.go

@@ -82,7 +82,7 @@ func setupWebhookControllers(mgr ctrl.Manager) error {
 	return nil
 }
 
-func setupCRDControllers(ctx context.Context, c client.Client, mgr ctrl.Manager) error {
+func setupCRDControllers(ctx context.Context, c client.Client, mgr ctrl.Manager, migrationChan chan struct{}) error {
 	ctrLog := setupLog.WithName("controller")
 	clusterTypeProvider, err := clustertype.NewProvider(ctx, c)
 
@@ -98,6 +98,7 @@ func setupCRDControllers(ctx context.Context, c client.Client, mgr ctrl.Manager)
 		Scheme:               mgr.GetScheme(),
 		ClusterTypeProvider:  clusterTypeProvider, // we want to cache information about the cluster type
 		StaticConfigProvider: staticInfoProvider,
+		MigrationCh:          migrationChan,
 	}).SetupWithManager(mgr, ctrLog.WithName("NicClusterPolicy")); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "NicClusterPolicy")
 		return err
@@ -166,35 +167,26 @@ func main() {
 		os.Exit(1)
 	}
 
-	// run migration logic before controllers start
-	if err := migrate.Migrate(stopCtx, setupLog.WithName("migrate"), directClient); err != nil {
-		setupLog.Error(err, "failed to run migration logic")
-		os.Exit(1)
+	migrationCompletionChan := make(chan struct{})
+	m := migrate.Migrator{
+		K8sClient:      directClient,
+		MigrationCh:    migrationCompletionChan,
+		LeaderElection: enableLeaderElection,
+		Logger:         ctrl.Log.WithName("Migrator"),
 	}
-
-	err = setupCRDControllers(stopCtx, directClient, mgr)
+	err = mgr.Add(&m)
 	if err != nil {
+		setupLog.Error(err, "failed to add Migrator to the Manager")
 		os.Exit(1)
 	}
 
-	upgrade.SetDriverName("ofed")
-
-	upgradeLogger := ctrl.Log.WithName("controllers").WithName("Upgrade")
-
-	clusterUpdateStateManager, err := upgrade.NewClusterUpgradeStateManager(
-		upgradeLogger.WithName("clusterUpgradeManager"), config.GetConfigOrDie(), nil)
-
+	err = setupCRDControllers(stopCtx, directClient, mgr, migrationCompletionChan)
 	if err != nil {
-		setupLog.Error(err, "unable to create new ClusterUpdateStateManager", "controller", "Upgrade")
 		os.Exit(1)
 	}
 
-	if err = (&controllers.UpgradeReconciler{
-		Client:       mgr.GetClient(),
-		Scheme:       mgr.GetScheme(),
-		StateManager: clusterUpdateStateManager,
-	}).SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "Upgrade")
+	err = setupUpgradeController(mgr, migrationCompletionChan)
+	if err != nil {
 		os.Exit(1)
 	}
 
@@ -221,3 +213,28 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func setupUpgradeController(mgr ctrl.Manager, migrationChan chan struct{}) error {
+	upgrade.SetDriverName("ofed")
+
+	upgradeLogger := ctrl.Log.WithName("controllers").WithName("Upgrade")
+
+	clusterUpdateStateManager, err := upgrade.NewClusterUpgradeStateManager(
+		upgradeLogger.WithName("clusterUpgradeManager"), config.GetConfigOrDie(), nil)
+
+	if err != nil {
+		setupLog.Error(err, "unable to create new ClusterUpdateStateManager", "controller", "Upgrade")
+		return err
+	}
+
+	if err = (&controllers.UpgradeReconciler{
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		StateManager: clusterUpdateStateManager,
+		MigrationCh:  migrationChan,
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Upgrade")
+		return err
+	}
+	return nil
+}

manifests/state-ofed-driver/0050_ofed-driver-ds.yaml

@@ -15,20 +15,22 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   labels:
-    app: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}
+    app: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}-{{ .RuntimeSpec.KernelHash }}
     nvidia.com/ofed-driver: ""
-  name: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}-ds
+    mofed-ds-format-version: "1"
+  name: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}-{{ .RuntimeSpec.KernelHash }}-ds
   namespace: {{ .RuntimeSpec.Namespace }}
 spec:
   updateStrategy:
     type: OnDelete
   selector:
     matchLabels:
-      app: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}
+      app: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}-{{ .RuntimeSpec.KernelHash }}
   template:
     metadata:
       labels:
-        app: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}
+        app: mofed-{{ .RuntimeSpec.OSName }}{{ .RuntimeSpec.OSVer }}-{{ .RuntimeSpec.KernelHash }}
+        kernel: {{ .RuntimeSpec.Kernel }}
         nvidia.com/ofed-driver: ""
     spec:
       priorityClassName: system-node-critical
@@ -242,6 +244,10 @@ spec:
         feature.node.kubernetes.io/pci-15b3.present: "true"
         feature.node.kubernetes.io/system-os_release.ID: {{ .RuntimeSpec.OSName }}
         feature.node.kubernetes.io/system-os_release.VERSION_ID: "{{ .RuntimeSpec.OSVer }}"
+        feature.node.kubernetes.io/kernel-version.full: "{{ .RuntimeSpec.Kernel }}"
+        {{- if .RuntimeSpec.UseDtk }}
+        feature.node.kubernetes.io/system-os_release.OSTREE_VERSION: "{{ .RuntimeSpec.RhcosVersion }}"
+        {{- end }}
       {{- if .NodeAffinity }}
       affinity:
         nodeAffinity: