Add support for CDI in k8s-rdma-shared-dev-plugin deployments

Signed-off-by: amaslennikov <amaslennikov@nvidia.com>
Mellanox · Sep 26, 2023 · a76c373 · a76c373
1 parent 578178f
commit a76c373
Show file tree

Hide file tree

Showing 12 changed files with 53 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -129,8 +129,8 @@ spec:
       periodSeconds: 30
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |
@@ -189,8 +189,8 @@ spec:
       periodSeconds: 30
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |

diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go
@@ -157,6 +157,7 @@ type DrainSpec struct {
 // 2. Device plugin configuration
 type DevicePluginSpec struct {
 	ImageSpecWithConfig `json:""`
+	UseCdi              bool `json:"useCdi,omitempty"`
 }
 
 // MultusSpec describes configuration options for Multus CNI

diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml
@@ -582,6 +582,8 @@ spec:
                   repository:
                     pattern: '[a-zA-Z0-9\.\-\/]+'
                     type: string
+                  useCdi:
+                    type: boolean
                   version:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
@@ -698,6 +700,8 @@ spec:
                   repository:
                     pattern: '[a-zA-Z0-9\.\-\/]+'
                     type: string
+                  useCdi:
+                    type: boolean
                   version:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string

diff --git a/config/samples/mellanox.com_v1alpha1_nicclusterpolicy.yaml b/config/samples/mellanox.com_v1alpha1_nicclusterpolicy.yaml
@@ -41,8 +41,8 @@ spec:
       maxParallelUpgrades: 1
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |

diff --git a/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml
@@ -582,6 +582,8 @@ spec:
                   repository:
                     pattern: '[a-zA-Z0-9\.\-\/]+'
                     type: string
+                  useCdi:
+                    type: boolean
                   version:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
@@ -698,6 +700,8 @@ spec:
                   repository:
                     pattern: '[a-zA-Z0-9\.\-\/]+'
                     type: string
+                  useCdi:
+                    type: boolean
                   version:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string

diff --git a/deployment/network-operator/values.yaml b/deployment/network-operator/values.yaml
@@ -198,8 +198,8 @@ ofedDriver:
 rdmaSharedDevicePlugin:
   deploy: true
   image: k8s-rdma-shared-dev-plugin
-  repository: nvcr.io/nvidia/cloud-native
-  version: v1.3.2
+  repository: ghcr.io/mellanox
+  version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
   # imagePullSecrets: []
   # The following defines the RDMA resources in the cluster
   # it must be provided by the user when deploying the chart

diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml
@@ -31,8 +31,8 @@ spec:
       periodSeconds: 30
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |

diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml
@@ -31,8 +31,8 @@ spec:
       periodSeconds: 30
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |

diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ocp.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ocp.yaml
@@ -30,8 +30,8 @@ spec:
       maxParallelUpgrades: 1
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |

diff --git a/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml
@@ -31,8 +31,8 @@ spec:
       periodSeconds: 30
   rdmaSharedDevicePlugin:
     image: k8s-rdma-shared-dev-plugin
-    repository: nvcr.io/nvidia/cloud-native
-    version: v1.3.2
+    repository: ghcr.io/mellanox
+    version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
     # The config below directly propagates to k8s-rdma-shared-device-plugin configuration.
     # Replace 'devices' with your (RDMA capable) netdevice name.
     config: |

diff --git a/hack/release.yaml b/hack/release.yaml
@@ -23,8 +23,8 @@ Mofed:
   version: 23.07-0.4.5.0
 RdmaSharedDevicePlugin:
   image: k8s-rdma-shared-dev-plugin
-  repository: nvcr.io/nvidia/cloud-native
-  version: v1.3.2
+  repository: ghcr.io/mellanox
+  version: fe7f371c7e1b8315bf900f71cd25cfc1251dc775
 SriovDevicePlugin:
   image: sriov-network-device-plugin
   repository: ghcr.io/k8snetworkplumbingwg

diff --git a/manifests/state-rdma-device-plugin/0060_rdma-shared-dev-plugin-ds.yaml b/manifests/state-rdma-device-plugin/0060_rdma-shared-dev-plugin-ds.yaml
@@ -54,6 +54,10 @@ spec:
       containers:
       - image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }}
         name: rdma-shared-dp
+        command: [ "/bin/k8s-rdma-shared-dp" ]
+        {{- if .CrSpec.UseCdi }}
+        args: [ "--use-cdi" ]
+        {{- end }}
         imagePullPolicy: IfNotPresent
         securityContext:
           privileged: true
@@ -64,6 +68,14 @@ spec:
             mountPath: /k8s-rdma-shared-dev-plugin
           - name: devs
             mountPath: /dev/
+          {{- if .CrSpec.UseCdi }}
+          - name: default-cdi
+            mountPath: /etc/cdi/
+          - name: dynamic-cdi
+            mountPath: /var/run/cdi
+          - name: host-config-volume
+            mountPath: /host/etc/pcidp/
+          {{- end }}
       volumes:
         - name: device-plugin
           hostPath:
@@ -77,6 +89,20 @@ spec:
         - name: devs
           hostPath:
             path: /dev/
+        {{- if .CrSpec.UseCdi }}
+        - name: default-cdi
+          hostPath:
+            path: /etc/cdi
+            type: DirectoryOrCreate
+        - name: dynamic-cdi
+          hostPath:
+            path: /var/run/cdi
+            type: DirectoryOrCreate
+        - name: host-config-volume
+          hostPath:
+            path: /etc/pcidp
+            type: DirectoryOrCreate
+        {{- end }}
       nodeSelector:
         feature.node.kubernetes.io/pci-15b3.present: "true"
         network.nvidia.com/operator.mofed.wait: "false"