feat: Add supports for vDPA interface

Signed-off-by: Ziteng Liu <ziteng.liu@jaguarmicro.com>
smartxworks · Jan 10, 2024 · db3f654 · db3f654
1 parent 30780fe
commit db3f654
Show file tree

Hide file tree

Showing 8 changed files with 126 additions and 2 deletions.
diff --git a/cmd/virt-prerunner/main.go b/cmd/virt-prerunner/main.go
@@ -252,6 +252,18 @@ func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloud
 						vmConfig.Devices = append(vmConfig.Devices, &sriovDeviceConfig)
 					}
 				}
+			case iface.VDPA != nil:
+				for _, networkStatus := range networkStatusList {
+					if networkStatus.Interface == linkName && networkStatus.DeviceInfo != nil && networkStatus.DeviceInfo.Vdpa != nil {
+						vdpaDeviceConfig := cloudhypervisor.VdpaConfig{
+							Id:        iface.Name,
+							NumQueues: iface.VDPA.NumQueues,
+							Iommu:     iface.VDPA.IOMMU,
+							Path:      networkStatus.DeviceInfo.Vdpa.Path,
+						}
+						vmConfig.Vdpa = append(vmConfig.Vdpa, &vdpaDeviceConfig)
+					}
+				}
 			case iface.VhostUser != nil:
 				netConfig := cloudhypervisor.NetConfig{
 					Id:        iface.Name,

diff --git a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml
@@ -910,6 +910,13 @@ spec:
                           type: string
                         sriov:
                           type: object
+                        vdpa:
+                          properties:
+                            iommu:
+                              type: boolean
+                            numQueues:
+                              type: integer
+                          type: object
                         vhostUser:
                           type: object
                       required:

diff --git a/docs/interfaces_and_networks.md b/docs/interfaces_and_networks.md
@@ -83,6 +83,7 @@ Each interface should declare its type by defining one of the following fields:
 | `bridge`     | Connect using a linux bridge                    |
 | `masquerade` | Connect using iptables rules to NAT the traffic |
 | `sriov`      | Passthrough a SR-IOV PCI device via VFIO        |
+| `vdpa`      | Add a vhost-vdpa device |
 
 Each interface may also have additional configuration fields that modify properties "seen" inside guest instances, as listed below:
 
@@ -224,3 +225,63 @@ spec:
       multus:
         networkName: mellanox-sriov-25g
 ```
+### `vdpa` Mode
+Similar to the sriov mode, the following components need to be installed:
+- [Multus CNI](https://github.com/k8snetworkplumbingwg/multus-cni)
+- A vDPA device plugin
+- A vDPA CNI Plugin
+
+If your hardware vendor support vDPA over SR-IOV, you can use
+ the [SR-IOV Network Device Plugin](https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin), here is a sample configuration:
+ ```yaml
+ apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sriovdp-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+        "resourceList": [
+            {
+                "resourceName": "jaguar",
+                "resourcePrefix": "jaguarmicro.com",
+                "selectors": {
+                    "vendors": ["1f53"],
+                    "devices": ["1000"],
+                    "drivers": ["jaguar"],
+                    "vdpaType": "vhost"
+                }
+            }
+        ]
+    }
+ ```
+ If you don't need complex network configuration, the changes to enable the SR-IOV CNI to also manage vDPA interfaces are in this repository:
+
+ https://github.com/amorenoz/sriov-cni/tree/rfe/vdpa
+
+#### Start an vDPA VM
+
+To create a VM that will attach to the aforementioned network, refer to the following VM spec:
+
+```yaml
+apiVersion: virt.virtink.smartx.com/v1alpha1
+kind: VirtualMachine
+spec:
+  instance:
+    interfaces:
+      - name: pod
+      - name: vdpa1
+        vdpa: { numQueues: 9, iommu: False }
+      - name: vdpa2
+        vdpa: { numQueues: 9, iommu: False }
+  networks:
+    - name: pod
+      pod: {}
+    - name: vdpa1
+      multus:
+        networkName: offload-ovn1
+    - name: vdpa2
+      multus:
+        networkName: offload-ovn2
+```
diff --git a/pkg/apis/virt/v1alpha1/types.go b/pkg/apis/virt/v1alpha1/types.go
@@ -102,6 +102,7 @@ type InterfaceBindingMethod struct {
 	Bridge     *InterfaceBridge     `json:"bridge,omitempty"`
 	Masquerade *InterfaceMasquerade `json:"masquerade,omitempty"`
 	SRIOV      *InterfaceSRIOV      `json:"sriov,omitempty"`
+	VDPA       *InterfaceVDPA       `json:"vdpa,omitempty"`
 	VhostUser  *InterfaceVhostUser  `json:"vhostUser,omitempty"`
 }
 
@@ -115,6 +116,11 @@ type InterfaceMasquerade struct {
 type InterfaceSRIOV struct {
 }
 
+type InterfaceVDPA struct {
+	NumQueues int  `json:"numQueues,omitempty"`
+	IOMMU     bool `json:"iommu,omitempty"`
+}
+
 type InterfaceVhostUser struct {
 }
 

diff --git a/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/controller/vm_webhook.go b/pkg/controller/vm_webhook.go
@@ -165,7 +165,7 @@ func MutateVM(ctx context.Context, vm *virtv1alpha1.VirtualMachine, oldVM *virtv
 			vm.Spec.Instance.Interfaces[i].MAC = macStr
 		}
 
-		if vm.Spec.Instance.Interfaces[i].Bridge == nil && vm.Spec.Instance.Interfaces[i].Masquerade == nil && vm.Spec.Instance.Interfaces[i].SRIOV == nil && vm.Spec.Instance.Interfaces[i].VhostUser == nil {
+		if vm.Spec.Instance.Interfaces[i].Bridge == nil && vm.Spec.Instance.Interfaces[i].Masquerade == nil && vm.Spec.Instance.Interfaces[i].SRIOV == nil && vm.Spec.Instance.Interfaces[i].VDPA == nil && vm.Spec.Instance.Interfaces[i].VhostUser == nil {
 			vm.Spec.Instance.Interfaces[i].InterfaceBindingMethod = virtv1alpha1.InterfaceBindingMethod{
 				Bridge: &virtv1alpha1.InterfaceBridge{},
 			}
@@ -504,6 +504,12 @@ func ValidateInterfaceBindingMethod(ctx context.Context, bindingMethod *virtv1al
 			errs = append(errs, field.Forbidden(fieldPath.Child("sriov"), "may not specify more than 1 binding method"))
 		}
 	}
+	if bindingMethod.VDPA != nil {
+		cnt++
+		if cnt > 1 {
+			errs = append(errs, field.Forbidden(fieldPath.Child("vdpa"), "may not specify more than 1 binding method"))
+		}
+	}
 	if bindingMethod.VhostUser != nil {
 		cnt++
 		if cnt > 1 {

diff --git a/pkg/controller/vm_webhook_test.go b/pkg/controller/vm_webhook_test.go
@@ -365,6 +365,17 @@ func TestValidateVM(t *testing.T) {
 			return vm
 		}(),
 		invalidFields: []string{"spec.instance.interfaces[0].sriov"},
+	}, {
+		vm: func() *virtv1alpha1.VirtualMachine {
+			vm := validVM.DeepCopy()
+			vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Bridge = nil
+			vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.VDPA = &virtv1alpha1.InterfaceVDPA{
+				NumQueues: 9,
+				IOMMU:     false,
+			}
+			return vm
+		}(),
+		invalidFields: []string{"spec.instance.interfaces[0].vdpa.numQueues", "spec.instance.interfaces[0].vdpa.iommu"},
 	}, {
 		vm: func() *virtv1alpha1.VirtualMachine {
 			vm := validVM.DeepCopy()

diff --git a/pkg/daemon/vm_controller.go b/pkg/daemon/vm_controller.go
@@ -145,7 +145,7 @@ func (r *VMReconciler) reconcile(ctx context.Context, vm *virtv1alpha1.VirtualMa
 				return err
 			}
 
-			if len(vmConfig.Devices) > 0 {
+			if len(vmConfig.Devices) > 0 || len(vmConfig.Vdpa) > 0 {
 				cloudHypervisorPID, err := pid.GetPIDBySocket(filepath.Join(getVMDataDirPath(vm), "ch.sock"))
 				if err != nil {
 					return fmt.Errorf("get cloud-hypervisor process pid: %s", err)