[DNM] chore: Bump multus version thick plugin

deployment/network-operator/values.yaml

@@ -362,9 +362,9 @@ secondaryNetwork:
     #       memory: "50Mi"
   multus:
     deploy: true
-    image: multus-cni
+    image: multus-cni@sha256
     repository: ghcr.io/k8snetworkplumbingwg
-    version: v3.9.3
+    version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8
     # imagePullSecrets: []
     # config: ''
     # containerResources:

example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-ipoib.yaml

@@ -66,9 +66,9 @@ spec:
       repository: ghcr.io/mellanox
       version: v1.2.0
     multus:
-      image: multus-cni
+      image: multus-cni@sha256
       repository: ghcr.io/k8snetworkplumbingwg
-      version: v3.9.3
+      version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8
       config: ''
     ipamPlugin:
       image: whereabouts

example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr-nvidia-ipam.yaml

@@ -62,9 +62,9 @@ spec:
       repository: ghcr.io/k8snetworkplumbingwg
       version: v1.5.0
     multus:
-      image: multus-cni
+      image: multus-cni@sha256
       repository: ghcr.io/k8snetworkplumbingwg
-      version: v3.9.3
+      version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8
       config: ''
   nvIpam:
     image: nvidia-k8s-ipam

example/crs/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml

@@ -62,9 +62,9 @@ spec:
       repository: ghcr.io/k8snetworkplumbingwg
       version: v1.5.0
     multus:
-      image: multus-cni
+      image: multus-cni@sha256
       repository: ghcr.io/k8snetworkplumbingwg
-      version: v3.9.3
+      version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8
       config: ''
     ipamPlugin:
       image: whereabouts

hack/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml

@@ -23,7 +23,7 @@ spec:
     singular: network-attachment-definition
     kind: NetworkAttachmentDefinition
     shortNames:
-    - net-attach-def
+      - net-attach-def
   versions:
     - name: v1
       served: true

hack/release.yaml

@@ -46,9 +46,9 @@ CniPlugins:
   repository: ghcr.io/k8snetworkplumbingwg
   version: v1.5.0
 Multus:
-  image: multus-cni
+  image: multus-cni@sha256
   repository: ghcr.io/k8snetworkplumbingwg
-  version: v3.9.3
+  version: ce1f91d6b49cb27bd0b92ac1c092727f0e5eca515728d994bfeda11e8b814cb8
 Ipoib:
   image: ipoib-cni
   repository: ghcr.io/mellanox

manifests/state-multus-cni/0010-cluter_role.yml

@@ -34,6 +34,8 @@ rules:
       - pods/status
     verbs:
       - get
+      - list
+      - watch
       - update
   - apiGroups:
       - ""

manifests/state-multus-cni/0040-configmap.yml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{ if .CrSpec.Config -}}
 kind: ConfigMap
 apiVersion: v1
 metadata:
@@ -21,5 +20,18 @@ metadata:
     tier: node
     app: multus
 data:
-  cni-conf.json: '{{ .CrSpec.Config }}'
-{{ end -}}
+{{- if .CrSpec.Config }}
+  daemon-config.json: '{{ .CrSpec.Config }}'
+{{- else }}
+  daemon-config.json: |
+    {
+        "chrootDir": "/hostroot",
+        "cniVersion": "0.3.1",
+        "logLevel": "verbose",
+        "logToStderr": true,
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusAutoconfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "socketDir": "/host/run/multus/"
+    }
+{{- end }}

manifests/state-multus-cni/0050-multus-ds.yml

@@ -19,8 +19,13 @@ spec:
         tier: node
         app: multus
         name: multus
+      annotations:
+        cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false"
     spec:
       hostNetwork: true
+      hostPID: true
+      priorityClassName: "system-node-critical"
+      terminationGracePeriodSeconds: 10
       {{- if .NodeAffinity }}
       affinity:
         nodeAffinity:
@@ -40,21 +45,31 @@ spec:
         - key: nvidia.com/gpu
           operator: Exists
           effect: NoSchedule
+      initContainers:
+        - name: install-multus-binary
+          image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }}
+          command:
+            - "cp"
+            - "-f"
+            - "/usr/src/multus-cni/bin/multus-shim"
+            - "/host/opt/cni/bin/multus-shim"
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "15Mi"
+          securityContext:
+            privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+              mountPropagation: Bidirectional
       containers:
         - name: kube-multus
           image: {{ .CrSpec.Repository }}/{{ .CrSpec.Image }}:{{ .CrSpec.Version }}
-          command: ["/entrypoint.sh"]
+          command: ["/usr/src/multus-cni/bin/multus-daemon"]
           args:
-            - "--cni-version=0.3.1"
-            # /tmp/multus-conf/00-multus.conf is where multus-cfg ConfigMap is mounted then entrypoint.sh copy it to
-            # /host/etc/cni/net.d/00-multus.conf
-            - "--multus-conf-file={{- if .CrSpec.Config -}}/tmp/multus-conf/00-multus.conf{{- else -}}auto{{- end -}}"
-          # Remove multus config file to prevent failing of creating/deleting pods since multus will fail due to
-          # permission issue, https://github.com/intel/multus-cni/issues/592
-          lifecycle:
-            preStop:
-              exec:
-                command: ["/bin/sh", "-c", "rm -f /host/etc/cni/net.d/00-multus.conf"]
+            - "--config=/etc/cni/net.d/multus.d/daemon-config.json"
           {{- with .RuntimeSpec.ContainerResources }}
           {{- with index . "kube-multus" }}
           resources:
@@ -78,27 +93,67 @@ spec:
           {{- end }}
           securityContext:
             privileged: true
+          terminationMessagePolicy: FallbackToLogsOnError
           volumeMounts:
             - name: cni
               mountPath: /host/etc/cni/net.d
+            # multus-daemon expects that cnibin path must be identical between pod and container host.
+            # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon,
+            # not to any other directory, like '/opt/bin' or '/usr/bin'.
             - name: cnibin
-              mountPath: /host/opt/cni/bin
-          {{- if .CrSpec.Config }}
-            - name: multus-cfg
-              mountPath: /tmp/multus-conf
-          {{- end }}
+              mountPath: /opt/cni/bin
+            - name: host-run
+              mountPath: /host/run
+            - name: host-var-lib-cni-multus
+              mountPath: /var/lib/cni/multus
+            - name: host-var-lib-kubelet
+              mountPath: /var/lib/kubelet
+              mountPropagation: HostToContainer
+            - name: host-run-k8s-cni-cncf-io
+              mountPath: /run/k8s.cni.cncf.io
+            - name: host-run-netns
+              mountPath: /run/netns
+              mountPropagation: HostToContainer
+            - name: multus-cni-config
+              mountPath: /etc/cni/net.d/multus.d
+              readOnly: true
+            - name: hostroot
+              mountPath: /hostroot
+              mountPropagation: HostToContainer
+          env:
+            - name: MULTUS_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
       volumes:
         - name: cni
           hostPath:
             path: /etc/cni/net.d
         - name: cnibin
           hostPath:
             path: {{ .RuntimeSpec.CniBinDirectory }}
-      {{- if .CrSpec.Config }}
-        - name: multus-cfg
+        - name: hostroot
+          hostPath:
+            path: /
+        - name: multus-cni-config
           configMap:
             name: multus-cni-config
             items:
-              - key: cni-conf.json
-                path: 00-multus.conf
-      {{- end }}
+            - key: daemon-config.json
+              path: daemon-config.json
+        - name: host-run
+          hostPath:
+            path: /run
+        - name: host-var-lib-cni-multus
+          hostPath:
+            path: /var/lib/cni/multus
+        - name: host-var-lib-kubelet
+          hostPath:
+            path: /var/lib/kubelet
+        - name: host-run-k8s-cni-cncf-io
+          hostPath:
+            path: /run/k8s.cni.cncf.io
+        - name: host-run-netns
+          hostPath:
+            path: /run/netns/
+

pkg/state/state_multus_cni_test.go

@@ -18,6 +18,7 @@ package state
 
 import (
 	"context"
+	"encoding/json"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -256,12 +257,8 @@ var _ = Describe("Multus CNI state", func() {
 		})).To(BeTrue())
 	})
 
-	It("should render resources correctly when config is specified in CR", func() {
+	It("should render config map with expected key", func() {
 		cr := getMinimalNicClusterPolicyWithMultus()
-
-		configString := "myconfig"
-		cr.Spec.SecondaryNetwork.Multus.Config = &configString
-
 		objs, err := state.GetManifestObjects(context.TODO(), cr, catalog, testLogger)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -271,53 +268,33 @@ var _ = Describe("Multus CNI state", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			Expect(configMap.Namespace).To(Equal(networkOperatorResourceNamespace))
-			Expect(configMap.Data["cni-conf.json"]).To(Equal(configString))
-		})).To(BeTrue())
-
-		Expect(runFuncForObjectInSlice(objs, "DaemonSet", func(obj *unstructured.Unstructured) {
-			var daemonSet appsv1.DaemonSet
-			err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &daemonSet)
-			Expect(err).NotTo(HaveOccurred())
-
-			Expect(daemonSet.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(
-				corev1.VolumeMount{
-					Name:      "multus-cfg",
-					MountPath: "/tmp/multus-conf",
-				},
-			))
-
-			Expect(daemonSet.Spec.Template.Spec.Volumes).To(ContainElement(
-				corev1.Volume{
-					Name: "multus-cfg",
-					VolumeSource: corev1.VolumeSource{
-						ConfigMap: &corev1.ConfigMapVolumeSource{
-							LocalObjectReference: corev1.LocalObjectReference{
-								Name: "multus-cni-config",
-							},
-							Items: []corev1.KeyToPath{
-								{
-									Key:  "cni-conf.json",
-									Path: "00-multus.conf",
-								},
-							},
-						},
-					},
-				},
-			))
-
+			defaultConfig := make(map[string]interface{})
+			Expect(configMap.Data).To(HaveKey("daemon-config.json"))
+			// just make sure data is valid json
+			Expect(json.Unmarshal([]byte(configMap.Data["daemon-config.json"]), &defaultConfig)).ToNot(HaveOccurred())
+			Expect(defaultConfig).ToNot(BeEmpty())
 		})).To(BeTrue())
 	})
 
-	It("should not render ConfigMap if config is not specified in CR", func() {
+	It("should render config map with config as specified in CR", func() {
 		cr := getMinimalNicClusterPolicyWithMultus()
+
+		configString := "myconfig"
+		cr.Spec.SecondaryNetwork.Multus.Config = &configString
+
 		objs, err := state.GetManifestObjects(context.TODO(), cr, catalog, testLogger)
 		Expect(err).NotTo(HaveOccurred())
 
-		for _, obj := range objs {
-			Expect(obj.GetKind()).ToNot(Equal("ConfigMap"))
-		}
-	})
+		Expect(runFuncForObjectInSlice(objs, "ConfigMap", func(obj *unstructured.Unstructured) {
+			var configMap corev1.ConfigMap
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &configMap)
+			Expect(err).NotTo(HaveOccurred())
 
+			Expect(configMap.Namespace).To(Equal(networkOperatorResourceNamespace))
+			Expect(configMap.Data).To(HaveKey("daemon-config.json"))
+			Expect(configMap.Data["daemon-config.json"]).To(Equal(configString))
+		})).To(BeTrue())
+	})
 })
 
 func getMinimalNicClusterPolicyWithMultus() *mellanoxv1alpha1.NicClusterPolicy {