DTK

Signed-off-by: Fred Rolland <frolland@nvidia.com>

api/v1alpha1/nicclusterpolicy_types.go

@@ -91,6 +91,9 @@ type OFEDDriverSpec struct {
 	// +kubebuilder:default:=300
 	// +kubebuilder:validation:Minimum:=0
 	TerminationGracePeriodSeconds int64 `json:"terminationGracePeriodSeconds,omitempty"`
+	// UseOCPDriverToolkit indicates if DriverToolkit image should be used on OpenShift to build and install driver modules
+	// +kubebuilder:default:=true
+	UseOCPDriverToolkit bool `json:"useOcpDriverToolkit"`
 }
 
 // DriverUpgradePolicySpec describes policy configuration for automatic upgrades

config/crd/bases/mellanox.com_nicclusterpolicies.yaml

@@ -573,12 +573,18 @@ spec:
                             type: integer
                         type: object
                     type: object
+                  useOcpDriverToolkit:
+                    default: true
+                    description: UseOCPDriverToolkit indicates if DriverToolkit image
+                      should be used on OpenShift to build and install driver modules
+                    type: boolean
                   version:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
                 - image
                 - repository
+                - useOcpDriverToolkit
                 - version
                 type: object
               psp:

config/rbac/role.yaml

@@ -181,6 +181,14 @@ rules:
   - create
   - patch
   - update
+- apiGroups:
+  - image.openshift.io
+  resources:
+  - imagestreams
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - k8s.cni.cncf.io
   resources:

controllers/nicclusterpolicy_controller.go

@@ -82,6 +82,7 @@ type NicClusterPolicyReconciler struct {
 // +kubebuilder:rbac:groups=nv-ipam.nvidia.com,resources=ippools/status,verbs=get;update;patch;
 // +kubebuilder:rbac:groups=cert-manager.io,resources=issuers;certificates,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=image.openshift.io,resources=imagestreams,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

deployment/network-operator/README.md

@@ -428,6 +428,7 @@ imagePullSecrets:
 | `ofedDriver.livenessProbe.periodSeconds` | int | 30 | Mellanox OFED liveness probe interval                                                                                                                                     |
 | `ofedDriver.readinessProbe.initialDelaySeconds` | int | 10 | Mellanox OFED readiness probe initial delay                                                                                                                               |
 | `ofedDriver.readinessProbe.periodSeconds` | int | 30 | Mellanox OFED readiness probe interval                                                                                                                                    |
+| `ofedDriver.useOcpDriverToolkit` | bool | `true` | In OpenShift, use Driver Toolkit image to compile OFED drivers                                                                                                                                    |
 
 #### RDMA Device Plugin
 

deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml

@@ -573,12 +573,18 @@ spec:
                             type: integer
                         type: object
                     type: object
+                  useOcpDriverToolkit:
+                    default: true
+                    description: UseOCPDriverToolkit indicates if DriverToolkit image
+                      should be used on OpenShift to build and install driver modules
+                    type: boolean
                   version:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
                 - image
                 - repository
+                - useOcpDriverToolkit
                 - version
                 type: object
               psp:

deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml

@@ -34,6 +34,7 @@ spec:
     image: {{ .Values.ofedDriver.image }}
     repository: {{ .Values.ofedDriver.repository }}
     version: {{ .Values.ofedDriver.version }}
+    useOcpDriverToolkit: {{ .Values.ofedDriver.useOcpDriverToolkit }}
     {{- if .Values.ofedDriver.env }}
     env:
       {{ toYaml .Values.ofedDriver.env | nindent 6 }}

deployment/network-operator/templates/role.yaml

@@ -195,6 +195,14 @@ rules:
   - create
   - patch
   - update
+- apiGroups:
+  - image.openshift.io
+  resources:
+  - imagestreams
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - k8s.cni.cncf.io
   resources:

deployment/network-operator/values.yaml

@@ -208,6 +208,7 @@ ofedDriver:
       # It's recommended to set a timeout to avoid infinite drain in case non-fatal error keeps happening on retries
       timeoutSeconds: 300
       deleteEmptyDir: false
+  useOcpDriverToolkit: true
 
 rdmaSharedDevicePlugin:
   deploy: true

hack/templates/values/values.template

@@ -208,6 +208,7 @@ ofedDriver:
       # It's recommended to set a timeout to avoid infinite drain in case non-fatal error keeps happening on retries
       timeoutSeconds: 300
       deleteEmptyDir: false
+  useOcpDriverToolkit: true
 
 rdmaSharedDevicePlugin:
   deploy: true

main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/NVIDIA/k8s-operator-libs/pkg/upgrade"
 	netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	osconfigv1 "github.com/openshift/api/config/v1"
+	imagev1 "github.com/openshift/api/image/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -55,6 +56,7 @@ func init() {
 	utilruntime.Must(mellanoxcomv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(netattdefv1.AddToScheme(scheme))
 	utilruntime.Must(osconfigv1.AddToScheme(scheme))
+	utilruntime.Must(imagev1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
 

manifests/state-ofed-driver/0050_ofed-driver-ds.yaml

@@ -54,6 +54,10 @@ spec:
         - image: {{ .RuntimeSpec.MOFEDImageName }}
           imagePullPolicy: IfNotPresent
           name: mofed-container
+          {{- if .RuntimeSpec.UseDtk }}
+          command: ["ocp_dtk_entrypoint"]
+          args: ["nv-fs-ctr-run-with-dtk"]
+          {{- end }}
           securityContext:
             privileged: true
             seLinuxOptions:
@@ -86,6 +90,10 @@ spec:
               readOnly: {{ .ReadOnly }}
             {{- end }}
             {{- end }}
+            {{- if .RuntimeSpec.UseDtk }}
+            - name: shared-nvidia-driver-toolkit
+              mountPath: /mnt/shared-nvidia-driver-toolkit
+            {{- end}}
           startupProbe:
             exec:
               command:
@@ -109,6 +117,47 @@ spec:
             initialDelaySeconds: {{ .CrSpec.ReadinessProbe.InitialDelaySeconds }}
             failureThreshold: 1
             periodSeconds: {{ .CrSpec.ReadinessProbe.PeriodSeconds }}
+        {{- if .RuntimeSpec.UseDtk }}
+        - image: {{ .RuntimeSpec.DtkImageName }}
+          imagePullPolicy: IfNotPresent
+          name: openshift-driver-toolkit-ctr
+          command: [bash, -xc]
+          args: ["until [ -f /mnt/shared-nvidia-driver-toolkit/dir_prepared ]; do echo  Waiting for nvidia-driver-ctr container to prepare the shared directory ...; sleep 10; done; exec /mnt/shared-nvidia-driver-toolkit/ocp_dtk_entrypoint dtk-build-driver"]
+          securityContext:
+            privileged: true
+            seLinuxOptions:
+              level: "s0"
+          env:
+          {{- if .CrSpec.Env }}
+          {{- range .CrSpec.Env }}
+            {{ . | yaml | nindentPrefix 14 "- " }}
+          {{- end }}
+          {{- end }}
+          volumeMounts:
+            - name: run-mlnx-ofed
+              mountPath: /run/mellanox/drivers
+              mountPropagation: Bidirectional
+            - name: etc-network
+              mountPath: /etc/network
+            - name: host-etc
+              mountPath: /host/etc
+            - name: host-usr
+              mountPath: /host/usr
+            - name: host-udev
+              mountPath: /host/lib/udev
+            - name: host-lib-modules
+              mountPath: /host/lib/modules
+            {{- if.AdditionalVolumeMounts.VolumeMounts }}
+            {{- range .AdditionalVolumeMounts.VolumeMounts }}
+            - name: {{ .Name }}
+              mountPath: {{ .MountPath }}
+              subPath: {{ .SubPath }}
+              readOnly: {{ .ReadOnly }}
+            {{- end }}
+            {{- end }}
+            - name: shared-nvidia-driver-toolkit
+              mountPath: /mnt/shared-nvidia-driver-toolkit
+        {{- end }}
       # unloading OFED modules can take more time than default terminationGracePeriod (30 sec)
       terminationGracePeriodSeconds: {{ .CrSpec.TerminationGracePeriodSeconds }}
       volumes:
@@ -139,12 +188,19 @@ spec:
             {{ . | yaml | nindentPrefix 14 "- " }}
             {{- end }}
         {{- end }}
+        {{- if .RuntimeSpec.UseDtk }}
+        - name: shared-nvidia-driver-toolkit
+          emptyDir: {}
+        {{- end }}
       nodeSelector:
         feature.node.kubernetes.io/pci-15b3.present: "true"
         feature.node.kubernetes.io/system-os_release.ID: {{ .RuntimeSpec.OSName }}
         feature.node.kubernetes.io/system-os_release.VERSION_ID: "{{ .RuntimeSpec.OSVer }}"
         feature.node.kubernetes.io/kernel-version.major: "{{ .RuntimeSpec.KernelMajor }}"
         feature.node.kubernetes.io/kernel-version.minor: "{{ .RuntimeSpec.KernelMinor }}"
+        {{- if .RuntimeSpec.UseDtk }}
+        feature.node.kubernetes.io/system-os_release.OSTREE_VERSION: "{{ .RuntimeSpec.RhcosVersion }}"
+        {{- end }}
       {{- if .NodeAffinity }}
       affinity:
         nodeAffinity:

pkg/state/state_ofed.go

@@ -27,6 +27,7 @@ import (
 	"github.com/Masterminds/semver/v3"
 	"github.com/go-logr/logr"
 	osconfigv1 "github.com/openshift/api/config/v1"
+	apiimagev1 "github.com/openshift/api/image/v1"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -147,7 +148,10 @@ type ofedRuntimeSpec struct {
 	KernelMinor    string
 	MOFEDImageName string
 	// is true if cluster type is Openshift
-	IsOpenshift bool
+	IsOpenshift  bool
+	UseDtk       bool
+	DtkImageName string
+	RhcosVersion string
 }
 
 type ofedManifestRenderData struct {
@@ -381,40 +385,44 @@ func (s *stateOFED) getManifestObjects(
 		reqLogger.V(consts.LogLevelInfo).Info("No nodes with Mellanox NICs where found in the cluster.")
 		return []*unstructured.Unstructured{}, nil
 	}
+	useDtk := clusterInfo.IsOpenshift() && cr.Spec.OFEDDriver.UseOCPDriverToolkit
+	rhcosDriverToolkitImages := make(map[string]string)
+	if useDtk {
+		images, err := s.ocpDriverToolkitImages(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get OpenShift DTK ImageStream : %v", err)
+		}
+		rhcosDriverToolkitImages = images
+	}
 	objs := make([]*unstructured.Unstructured, 0)
 	for _, np := range nodePools {
 		nodePool := np
 		setProbesDefaults(cr)
-
 		// Update MOFED Env variables with defaults for the cluster
 		cr.Spec.OFEDDriver.Env = s.mergeWithDefaultEnvs(cr.Spec.OFEDDriver.Env, &nodePool)
 
 		additionalVolMounts := additionalVolumeMounts{}
 		osname := nodePool.OsName
-		// set any custom ssl key/certificate configuration provided
-		if cr.Spec.OFEDDriver.CertConfig != nil && cr.Spec.OFEDDriver.CertConfig.Name != "" {
-			destinationDir, err := getCertConfigPath(osname)
-			if err != nil {
-				return nil, fmt.Errorf("failed to get destination directory for custom TLS certificates config: %v", err)
-			}
 
-			err = s.handleAdditionalMounts(ctx, &additionalVolMounts, cr.Spec.OFEDDriver.CertConfig.Name, destinationDir)
-			if err != nil {
-				return nil, fmt.Errorf("failed to mount volumes for custom TLS certificates: %v", err)
-			}
+		// set any custom ssl key/certificate configuration provided
+		err := s.handleCertConfig(ctx, cr, osname, additionalVolMounts)
+		if err != nil {
+			return nil, err
 		}
 
 		// set any custom repo configuration provided
-		if cr.Spec.OFEDDriver.RepoConfig != nil && cr.Spec.OFEDDriver.RepoConfig.Name != "" {
-			destinationDir, err := getRepoConfigPath(osname)
-			if err != nil {
-				return nil, fmt.Errorf("failed to get destination directory for custom repo config: %v", err)
-			}
+		err = s.handleRepoConfig(ctx, cr, osname, additionalVolMounts)
+		if err != nil {
+			return nil, err
+		}
 
-			err = s.handleAdditionalMounts(ctx, &additionalVolMounts, cr.Spec.OFEDDriver.RepoConfig.Name, destinationDir)
-			if err != nil {
-				return nil, fmt.Errorf("failed to mount volumes for custom repositories configuration: %v", err)
+		dtkImageName := ""
+		if useDtk {
+			image, ok := rhcosDriverToolkitImages[nodePool.RhcosVersion]
+			if !ok {
+				return nil, fmt.Errorf("failed to find DTK image for RHCOS version: %v", nodePool.RhcosVersion)
 			}
+			dtkImageName = image
 		}
 
 		renderData := &ofedManifestRenderData{
@@ -428,6 +436,9 @@ func (s *stateOFED) getManifestObjects(
 				KernelMinor:    nodePool.KernelMinor,
 				MOFEDImageName: s.getMofedDriverImageName(cr, &nodePool, reqLogger),
 				IsOpenshift:    clusterInfo.IsOpenshift(),
+				UseDtk:         useDtk,
+				DtkImageName:   dtkImageName,
+				RhcosVersion:   nodePool.RhcosVersion,
 			},
 			Tolerations:            cr.Spec.Tolerations,
 			NodeAffinity:           cr.Spec.NodeAffinity,
@@ -445,6 +456,38 @@ func (s *stateOFED) getManifestObjects(
 	return objs, nil
 }
 
+func (s *stateOFED) handleRepoConfig(
+	ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts additionalVolumeMounts) error {
+	if cr.Spec.OFEDDriver.RepoConfig != nil && cr.Spec.OFEDDriver.RepoConfig.Name != "" {
+		destinationDir, err := getRepoConfigPath(osname)
+		if err != nil {
+			return fmt.Errorf("failed to get destination directory for custom repo config: %v", err)
+		}
+
+		err = s.handleAdditionalMounts(ctx, &mounts, cr.Spec.OFEDDriver.RepoConfig.Name, destinationDir)
+		if err != nil {
+			return fmt.Errorf("failed to mount volumes for custom repositories configuration: %v", err)
+		}
+	}
+	return nil
+}
+
+func (s *stateOFED) handleCertConfig(
+	ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts additionalVolumeMounts) error {
+	if cr.Spec.OFEDDriver.CertConfig != nil && cr.Spec.OFEDDriver.CertConfig.Name != "" {
+		destinationDir, err := getCertConfigPath(osname)
+		if err != nil {
+			return fmt.Errorf("failed to get destination directory for custom TLS certificates config: %v", err)
+		}
+
+		err = s.handleAdditionalMounts(ctx, &mounts, cr.Spec.OFEDDriver.CertConfig.Name, destinationDir)
+		if err != nil {
+			return fmt.Errorf("failed to mount volumes for custom TLS certificates: %v", err)
+		}
+	}
+	return nil
+}
+
 func setProbesDefaults(cr *mellanoxv1alpha1.NicClusterPolicy) {
 	if cr.Spec.OFEDDriver.StartupProbe == nil {
 		cr.Spec.OFEDDriver.StartupProbe = &mellanoxv1alpha1.PodProbeSpec{
@@ -661,3 +704,22 @@ func (e envVarsWithGet) Get(name string) *v1.EnvVar {
 
 	return nil
 }
+
+func (s *stateOFED) ocpDriverToolkitImages(ctx context.Context) (map[string]string, error) {
+	reqLogger := log.FromContext(ctx)
+	dtkImageStream := &apiimagev1.ImageStream{}
+	name := "driver-toolkit"
+	namespace := "openshift"
+	err := s.client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, dtkImageStream)
+	if err != nil {
+		reqLogger.Error(err, "Couldn't get the driver-toolkit imagestream")
+		return nil, err
+	}
+	rhcosDriverToolkitImages := make(map[string]string)
+	reqLogger.Info("ocpDriverToolkitImages: driver-toolkit imagestream found")
+	for _, tag := range dtkImageStream.Spec.Tags {
+		rhcosDriverToolkitImages[tag.Name] = tag.From.Name
+	}
+
+	return rhcosDriverToolkitImages, nil
+}