pci: add pci device support in dbs-cli

1. attach pci device during the boot time of Dragonball.
2. hot plug pci device when the Dragonball is running.
3. prepare to hot unplug pci device when the Dragonball is running.
4. hot unplug pci device when the Dragonball is running.

fixes: #29

Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>

.gitignore

@@ -10,4 +10,8 @@
 **/*.rs.bk
 
 # log files
-**/*.log
+**/*.log
+
+.idea*
+
+.vscode*

Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-dragonball = { path = "/root/kata-containers/src/dragonball", features = [
+dragonball = { git = "https://github.com/kata-containers/kata-containers", branch = "main", features = [
     "virtio-blk",
     "virtio-fs",
     "virtio-vsock",
@@ -15,7 +15,7 @@ dragonball = { path = "/root/kata-containers/src/dragonball", features = [
     "hotplug",
     "dbs-upcall",
     "vhost-user-net",
-    "vhost-user-net"
+    "host-device"
 ] }
 clap = { version = "4.0.27", features = ["derive"] }
 serde = "1.0.27"

README.md

@@ -117,6 +117,30 @@ The supported network devices include:
 }
 ```
 
+### PCI Device
+
+You can choose to attach a pci device during the boot time of Dragonball. 
+Please note that hostdev_id and bus_slot_func are the must parameters for attaching pci device.  
+Please make sure that pci device is binded to `vfio-pci` driver.
+
+```
+./dbs-cli create --kernel-path $KERNEL_PATH --rootfs $ROOTFS_PATH --boot-args "console=ttyS0 tty0 reboot=k debug panic=1 root=/dev/vda1" --hostdev-id $HOST_DEVICE_ID --bus-slot-func $BUS_SLOT_FUNC
+```
+
+#### How to get hostdev_id and bus_slot_func?
+
+hostdev_id: This is an id you pick for each pci device attaching into VM. So name it whatever number you want.
+
+bus_slot_func: take a network device as the example, you could use `lspci | grep "network device"` and you could get something like
+```
+[root@xxx ~]# lspci | grep "network device"
+5d:00.0 Ethernet controller: Red Hat, Inc. Virtio network device
+5e:00.0 Ethernet controller: Red Hat, Inc. Virtio network device
+```
+`5d:00.0` is the bus_slot_func.
+
+As an alternative way to insert a host device, you can use upcall to hotplug / hot-unplug a pci device into Dragonball while Dragonball is running, for more details please go to advance usgae part of this document.
+
 ## Advanced Usage
 
 ### Create API Server and Update VM
@@ -153,6 +177,23 @@ sudo ./dbs-cli  \
   --hotplug-virblks '[{"drive_id":"testblk","device_type":"RawBlock","path_on_host":"/path/to/test.img","is_root_device":false,"is_read_only":false,"is_direct":false,"no_drop":false,"num_queues":1,"queue_size":1024}]' \
 ```
 
+Hotplug a pci device into Dragonball
+```
+./dbs-cli --api-sock-path $API_SOCK_PATH update --bus-slot-func $BUS_SLOT_FUNC --hostdev-id $HOST_DEVICE_ID
+```
+
+Prepare hot-unplug a pci device into Dragonball (must do before hotunplug)
+
+```
+./dbs-cli --api-sock-path ./sock update --prepare-remove-host-device $HOST_DEVICE_ID
+```
+
+Hot-unplug a pci device into Dragonball
+
+```
+./dbs-cli --api-sock-path ./sock update --remove-host-device $HOST_DEVICE_ID
+```
+
 TODO : add document for hot-plug virtio-fs
 
 ### Exit VM

src/api_client.rs

@@ -7,25 +7,45 @@ use std::os::unix::net::UnixStream;
 use anyhow::{Context, Result};
 use serde_json::{json, Value};
 
-use crate::parser::args::UpdateArgs;
+use crate::parser::args::{HostDeviceArgs, UpdateArgs};
 
 pub fn run_api_client(args: UpdateArgs, api_sock_path: &str) -> Result<()> {
     if let Some(vcpu_resize_num) = args.vcpu_resize {
         let request = request_cpu_resize(vcpu_resize_num);
         send_request(request, api_sock_path)?;
     }
+
     if let Some(config) = args.virnets {
         let request = request_virtio_net(&config);
         send_request(request, api_sock_path)?;
     }
+
     if let Some(config) = args.virblks {
         let request = request_virtio_blk(&config);
         send_request(request, api_sock_path)?;
     }
+
     if let Some(config) = args.patch_fs {
         let request = request_patch_fs(&config);
         send_request(request, api_sock_path)?;
     }
+
+    if let Some(host_device_args) = args.insert_host_device {
+        if host_device_args.bus_slot_func.is_some() {
+            let request = request_insert_host_device(host_device_args.clone());
+            send_request(request, api_sock_path)?;
+        }
+    }
+
+    if let Some(host_device_id) = args.prepare_remove_host_device {
+        let request = request_prepare_remove_host_device(host_device_id.clone());
+        send_request(request, api_sock_path)?;
+    }
+
+    if let Some(host_device_id) = args.remove_host_device {
+        let request = request_remove_host_device(host_device_id.clone());
+        send_request(request, api_sock_path)?;
+    }
     Ok(())
 }
 
@@ -59,6 +79,28 @@ fn request_patch_fs(patch_fs_config: &str) -> Value {
     })
 }
 
+fn request_insert_host_device(host_device_args: HostDeviceArgs) -> Value {
+    json!({
+        "action": "insert_host_device",
+        "hostdev-id": host_device_args.hostdev_id.expect("host device arg should be provided to insert host device."),
+        "bus-slot-func": host_device_args.bus_slot_func.expect("bus_slot_func should be provided to insert host device."),
+    })
+}
+
+fn request_prepare_remove_host_device(host_device_id: String) -> Value {
+    json!({
+        "action": "prepare_remove_host_device",
+        "hostdev-id": host_device_id.clone(),
+    })
+}
+
+fn request_remove_host_device(host_device_id: String) -> Value {
+    json!({
+        "action": "remove_host_device",
+        "hostdev-id": host_device_id.clone(),
+    })
+}
+
 fn send_request(request: Value, api_sock_path: &str) -> Result<()> {
     let mut unix_stream = UnixStream::connect(api_sock_path).context("Could not create stream")?;
 

src/api_server.rs

@@ -83,21 +83,26 @@ impl ApiServer {
             }
             Some("insert_host_device") => {
                 let host_device_config = HostDeviceConfig {
-                    hostdev_id: v["hostdev_id"].as_str().unwrap().to_owned(),
-                    sysfs_path: v["sysfs_path"].as_str().unwrap().to_owned(),
+                    hostdev_id: v["hostdev-id"].as_str().unwrap().to_owned(),
+                    sysfs_path: "".to_string(),
                     dev_config: VfioPciDeviceConfig {
-                        bus_slot_func: v["bus_slot_func"].as_str().unwrap().to_owned(),
-                        vendor_device_id: v["vendor_device_id"]
-                            .as_u64()
-                            .map(|vendor_device_id: u64| vendor_device_id as u32)
-                            .unwrap(),
+                        bus_slot_func: v["bus-slot-func"].as_str().unwrap().to_owned(),
+                        vendor_device_id: 0,
                         guest_dev_id: None,
                         clique_id: None,
                     },
                 };
                 self.insert_host_device(host_device_config)
                     .expect("Failed to insert a host device");
             }
+            Some("prepare_remove_host_device") => {
+                self.prepare_remove_host_device(v["hostdev-id"].as_str().unwrap().to_owned())
+                    .expect("Failed to insert a host device");
+            }
+            Some("remove_host_device") => {
+                self.remove_host_device(v["hostdev-id"].as_str().unwrap().to_owned())
+                    .expect("Failed to insert a host device");
+            }
             Some("insert_virnets") => {
                 let config_json = match v["config"].as_str() {
                     Some(config_json) => config_json,

src/cli_instance.rs

@@ -157,7 +157,8 @@ impl CliInstance {
                 .expect("failed to set vsock socket path");
         }
 
-        if !args.host_device.hostdev_id.is_none() {
+        // users should at least provide hostdev_id and bus_slot_func to insert a host device
+        if args.host_device.hostdev_id.is_some() && args.host_device.bus_slot_func.is_some() {
             let host_device_config =
                 HostDeviceConfig {
                     hostdev_id: args
@@ -167,14 +168,12 @@ impl CliInstance {
                     sysfs_path: args
                         .host_device
                         .sysfs_path
-                        .expect("There has to be sysfs_path if you want to add host device."),
+                        .unwrap_or_default(),
                     dev_config: VfioPciDeviceConfig {
                         bus_slot_func: args.host_device.bus_slot_func.expect(
                             "There has to be bus_slot_func if you want to add host device.",
                         ),
-                        vendor_device_id: args.host_device.vendor_device_id.expect(
-                            "There has to be vendor_device_id if you want to add host device.",
-                        ),
+                        vendor_device_id: args.host_device.vendor_device_id.unwrap_or_default(), 
                         guest_dev_id: args.host_device.guest_dev_id,
                         clique_id: args.host_device.clique_id,
                     },