From 866060ab3ed3f7fd88b20969eae7a045588b5769 Mon Sep 17 00:00:00 2001 From: Dan Milea Date: Wed, 19 Jul 2023 11:53:10 +0300 Subject: [PATCH] virtio: virtio-mmio framework VIRTIO MMIO transport for OpenAMP. Signed-off-by: Dan Milea --- cmake/options.cmake | 6 + lib/CMakeLists.txt | 3 + lib/include/openamp/rpmsg_virtio.h | 2 +- lib/include/openamp/virtio.h | 70 ++++- lib/include/openamp/virtio_mmio.h | 262 +++++++++++++++++++ lib/include/openamp/virtqueue.h | 26 ++ lib/virtio/virtio.c | 4 +- lib/virtio_mmio/CMakeLists.txt | 3 + lib/virtio_mmio/virtio_mmio_drv.c | 404 +++++++++++++++++++++++++++++ 9 files changed, 770 insertions(+), 10 deletions(-) create mode 100644 lib/include/openamp/virtio_mmio.h create mode 100644 lib/virtio_mmio/CMakeLists.txt create mode 100644 lib/virtio_mmio/virtio_mmio_drv.c diff --git a/cmake/options.cmake b/cmake/options.cmake index 4f895893f..a605f20ee 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -78,6 +78,12 @@ if (NOT WITH_VIRTIO_DEVICE AND NOT WITH_VIRTIO_SLAVE) add_definitions(-DVIRTIO_DRIVER_ONLY) endif (NOT WITH_VIRTIO_DEVICE AND NOT WITH_VIRTIO_SLAVE) +option (WITH_VIRTIO_MMIO_DRV "Build with virtio mmio driver support enabled" OFF) + +if (WITH_VIRTIO_MMIO_DRV) + add_definitions(-DWITH_VIRTIO_MMIO_DRV) +endif (WITH_VIRTIO_MMIO_DRV) + option (WITH_DCACHE_VRINGS "Build with vrings cache operations enabled" OFF) if (WITH_DCACHE_VRINGS) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index a36325689..ff2e39916 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -10,6 +10,9 @@ collect (PROJECT_LIB_SOURCES version.c) add_subdirectory (virtio) add_subdirectory (rpmsg) add_subdirectory (remoteproc) +if (WITH_VIRTIO_MMIO_DRV) +add_subdirectory (virtio_mmio) +endif (WITH_VIRTIO_MMIO_DRV) if (WITH_PROXY) add_subdirectory (proxy) diff --git a/lib/include/openamp/rpmsg_virtio.h b/lib/include/openamp/rpmsg_virtio.h index 3d03e8d5d..e4da54833 100644 --- a/lib/include/openamp/rpmsg_virtio.h +++ b/lib/include/openamp/rpmsg_virtio.h @@ -142,7 +142,7 @@ rpmsg_virtio_create_virtqueues(struct rpmsg_virtio_device *rvdev, vq_callback *callbacks) { return virtio_create_virtqueues(rvdev->vdev, flags, nvqs, names, - callbacks); + callbacks, NULL); } /** diff --git a/lib/include/openamp/virtio.h b/lib/include/openamp/virtio.h index 5e2f6427c..bad51731c 100644 --- a/lib/include/openamp/virtio.h +++ b/lib/include/openamp/virtio.h @@ -98,6 +98,7 @@ __deprecated static inline int deprecated_virtio_dev_slave(void) struct virtio_device_id { uint32_t device; uint32_t vendor; + uint32_t version; }; /* @@ -120,6 +121,44 @@ struct virtio_device_id { #define VIRTIO_TRANSPORT_F_START 28 #define VIRTIO_TRANSPORT_F_END 32 +#ifdef VIRTIO_DEBUG +#include + +#define VIRTIO_ASSERT(_exp, _msg) do { \ + if (!(_exp)) { \ + metal_log(METAL_LOG_EMERGENCY, \ + "FATAL: %s - " _msg, __func__); \ + metal_assert(_exp); \ + } \ + } while (0) +#else +#define VIRTIO_ASSERT(_exp, _msg) metal_assert(_exp) +#endif /* VIRTIO_DEBUG */ + +#define VRING_ALIGNMENT 4096 + +#define VIRTIO_RING_SIZE(n, align) \ + (( \ + ( \ + sizeof(struct vring_desc) * n + \ + sizeof(struct vring_avail) + \ + sizeof(uint16_t) * (n + 1) + \ + align - 1 \ + ) \ + & ~(align - 1) \ + ) + \ + sizeof(struct vring_used) + \ + sizeof(struct vring_used_elem) * n + sizeof(uint16_t)) + +#define VRING_DECLARE(name, n, align) \ +static char __vrbuf_##name[VIRTIO_RING_SIZE(n, align)] __aligned(VRING_ALIGNMENT); \ +static struct vring __vring_##name = { \ + .desc = (void *)__vrbuf_##name, \ + .avail = (void *)((unsigned long)__vrbuf_##name + n * sizeof(struct vring_desc)), \ + .used = (void *)((unsigned long)__vrbuf_##name + ((n * sizeof(struct vring_desc) + \ + (n + 1) * sizeof(uint16_t) + align - 1) & ~(align - 1))), \ +} + typedef void (*virtio_dev_reset_cb)(struct virtio_device *vdev); struct virtio_dispatch; @@ -180,7 +219,8 @@ struct virtio_dispatch { int (*create_virtqueues)(struct virtio_device *vdev, unsigned int flags, unsigned int nvqs, const char *names[], - vq_callback callbacks[]); + vq_callback callbacks[], + void *callback_args[]); void (*delete_virtqueues)(struct virtio_device *vdev); uint8_t (*get_status)(struct virtio_device *dev); void (*set_status)(struct virtio_device *dev, uint8_t status); @@ -205,17 +245,18 @@ struct virtio_dispatch { /** * @brief Create the virtio device virtqueue. * - * @param vdev Pointer to virtio device structure. - * @param flags Create flag. - * @param nvqs The virtqueue number. - * @param names Virtqueue names. - * @param callbacks Virtqueue callback functions. + * @param vdev Pointer to virtio device structure. + * @param flags Create flag. + * @param nvqs The virtqueue number. + * @param names Virtqueue names. + * @param callbacks Virtqueue callback functions. + * @param callback_args Virtqueue callback function arguments. * * @return 0 on success, otherwise error code. */ int virtio_create_virtqueues(struct virtio_device *vdev, unsigned int flags, unsigned int nvqs, const char *names[], - vq_callback callbacks[]); + vq_callback callbacks[], void *callback_args[]); /** * @brief Delete the virtio device virtqueue. @@ -236,6 +277,21 @@ static inline int virtio_delete_virtqueues(struct virtio_device *vdev) return 0; } +/** + * @brief Get device ID. + * + * @param dev Pointer to device structure. + * + * @return Device ID value. + */ + +static inline uint32_t virtio_get_devid(const struct virtio_device *vdev) +{ + if (!vdev) + return 0; + return vdev->id.device; +} + /** * @brief Retrieve device status. * diff --git a/lib/include/openamp/virtio_mmio.h b/lib/include/openamp/virtio_mmio.h new file mode 100644 index 000000000..95d6498ac --- /dev/null +++ b/lib/include/openamp/virtio_mmio.h @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2022 Wind River Systems, Inc. + * Based on Virtio PCI driver by Anthony Liguori, copyright IBM Corp. 2007 + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef OPENAMP_VIRTIO_MMIO_H +#define OPENAMP_VIRTIO_MMIO_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Enable support for legacy devices */ +#define VIRTIO_MMIO_LEGACY + +/* Control registers */ + +/* Magic value ("virt" string) - Read Only */ +#define VIRTIO_MMIO_MAGIC_VALUE 0x000 + +#define VIRTIO_MMIO_MAGIC_VALUE_STRING ('v' | ('i' << 8) | ('r' << 16) | ('t' << 24)) + +/* Virtio device version - Read Only */ +#define VIRTIO_MMIO_VERSION 0x004 + +/* Virtio device ID - Read Only */ +#define VIRTIO_MMIO_DEVICE_ID 0x008 + +/* Virtio vendor ID - Read Only */ +#define VIRTIO_MMIO_VENDOR_ID 0x00c + +/* + * Bitmask of the features supported by the device (host) + * (32 bits per set) - Read Only + */ +#define VIRTIO_MMIO_DEVICE_FEATURES 0x010 + +/* Device (host) features set selector - Write Only */ +#define VIRTIO_MMIO_DEVICE_FEATURES_SEL 0x014 + +/* + * Bitmask of features activated by the driver (guest) + * (32 bits per set) - Write Only + */ +#define VIRTIO_MMIO_DRIVER_FEATURES 0x020 + +/* Activated features set selector - Write Only */ +#define VIRTIO_MMIO_DRIVER_FEATURES_SEL 0x024 + +#ifndef VIRTIO_MMIO_NO_LEGACY /* LEGACY DEVICES ONLY! */ +/* Guest's memory page size in bytes - Write Only */ +#define VIRTIO_MMIO_GUEST_PAGE_SIZE 0x028 +#endif + +/* Queue selector - Write Only */ +#define VIRTIO_MMIO_QUEUE_SEL 0x030 + +/* Maximum size of the currently selected queue - Read Only */ +#define VIRTIO_MMIO_QUEUE_NUM_MAX 0x034 + +/* Queue size for the currently selected queue - Write Only */ +#define VIRTIO_MMIO_QUEUE_NUM 0x038 + +#ifdef VIRTIO_MMIO_LEGACY +/* Used Ring alignment for the currently selected queue - Write Only */ +#define VIRTIO_MMIO_QUEUE_ALIGN 0x03c +/* Guest's PFN for the currently selected queue - Read Write */ +#define VIRTIO_MMIO_QUEUE_PFN 0x040 +#endif + +/* Ready bit for the currently selected queue - Read Write */ +#define VIRTIO_MMIO_QUEUE_READY 0x044 + +/* Queue notifier - Write Only */ +#define VIRTIO_MMIO_QUEUE_NOTIFY 0x050 + +/* Interrupt status - Read Only */ +#define VIRTIO_MMIO_INTERRUPT_STATUS 0x060 + +/* Interrupt acknowledge - Write Only */ +#define VIRTIO_MMIO_INTERRUPT_ACK 0x064 + +/* Device status register - Read Write */ +#define VIRTIO_MMIO_STATUS 0x070 + +/* Selected queue's Descriptor Table address, 64 bits in two halves */ +#define VIRTIO_MMIO_QUEUE_DESC_LOW 0x080 +#define VIRTIO_MMIO_QUEUE_DESC_HIGH 0x084 + +/* Selected queue's Available Ring address, 64 bits in two halves */ +#define VIRTIO_MMIO_QUEUE_AVAIL_LOW 0x090 +#define VIRTIO_MMIO_QUEUE_AVAIL_HIGH 0x094 + +/* Selected queue's Used Ring address, 64 bits in two halves */ +#define VIRTIO_MMIO_QUEUE_USED_LOW 0x0a0 +#define VIRTIO_MMIO_QUEUE_USED_HIGH 0x0a4 + +/* Shared memory region id */ +#define VIRTIO_MMIO_SHM_SEL 0x0ac + +/* Shared memory region length, 64 bits in two halves */ +#define VIRTIO_MMIO_SHM_LEN_LOW 0x0b0 +#define VIRTIO_MMIO_SHM_LEN_HIGH 0x0b4 + +/* Shared memory region base address, 64 bits in two halves */ +#define VIRTIO_MMIO_SHM_BASE_LOW 0x0b8 +#define VIRTIO_MMIO_SHM_BASE_HIGH 0x0bc + +/* Configuration atomicity value */ +#define VIRTIO_MMIO_CONFIG_GENERATION 0x0fc + +/* + * The config space is defined by each driver as + * the per-driver configuration space - Read Write + */ +#define VIRTIO_MMIO_CONFIG 0x100 + +/* Interrupt flags (re: interrupt status & acknowledge registers) */ +#define VIRTIO_MMIO_INT_VRING (1 << 0) +#define VIRTIO_MMIO_INT_CONFIG (1 << 1) + +/* Data buffer size for preallocated buffers before vring */ +#define VIRTIO_MMIO_MAX_DATA_SIZE 128 + +/** + * @brief Declare a virtqueue structure. + * + * @param name The name of the virtqueue structure. + * @param n Size of the virtqueue. Must be a power of 2. + * @param align Memory alignment of the associated vring structures. + */ + +#define VIRTIO_MMIO_VQ_DECLARE(name, n, align) \ + static char __vrbuf_##name[VIRTIO_RING_SIZE(n, align)] __aligned(VRING_ALIGNMENT); \ + static struct { \ + struct virtqueue vq; \ + struct vq_desc_extra extra[n]; \ + } __vq_wrapper_##name = { \ + .vq = { \ + .vq_nentries = n, \ + .vq_ring = { \ + .desc = (void *)__vrbuf_##name, \ + .avail = (void *)((unsigned long)__vrbuf_##name + \ + n * sizeof(struct vring_desc)), \ + .used = (void *)((unsigned long)__vrbuf_##name + \ + ((n * sizeof(struct vring_desc) + \ + (n + 1) * sizeof(uint16_t) + align - 1) & ~(align - 1))), \ + }, \ + .vq_queued_cnt = 0, \ + .vq_free_cnt = n, \ + }, \ + } \ + /**< @hideinitializer */ + +/** + * @brief Retrieve a pointer to the virtqueue structure declared with VQ_DECLARE(). + * + * @param name The name of the virtqueue structure. + * + * @return A pointer to the virtqueue structure. + */ + +#define VIRTIO_MMIO_VQ_PTR(name) \ + (&__vq_wrapper_##name.vq) \ + /**< @hideinitializer */ + +/** + * struct virtio_mmio_dev_mem: VIRTIO MMIO memory area + * @base memory region physical address + * @size memory region size + */ +struct virtio_mmio_dev_mem { + void *base; + size_t size; +}; + +/** + * struct virtio_mmio_device: representation of a VIRTIO MMIO device + * @vdev base virtio device struct + * @cfg_io device configuration space metal_io_region + * @shm_io pre-shared memory space metal_io_region + * @shm_device shared memory device + * @cfg_mem VIRTIO device configuration space + * @shm_mem VIRTIO device pre-shared memory + * @device_mode VIRTIO_DEV_DRIVER or VIRTIO_DEV_DEVICE + * @irq interrupt number + * @user_data custom user data + */ +struct virtio_mmio_device { + struct virtio_device vdev; + struct metal_io_region *cfg_io; + struct metal_io_region *shm_io; + struct metal_device shm_device; + struct virtio_mmio_dev_mem cfg_mem; + struct virtio_mmio_dev_mem shm_mem; + unsigned int device_mode; + unsigned int irq; + void *user_data; +}; + +/** + * @brief Register a VIRTIO device with the VIRTIO stack. + * + * @param dev Pointer to device structure. + * @param vq_num Number of virtqueues the device uses. + * @param vqs Array of pointers to vthe virtqueues used by the device. + */ + +void virtio_mmio_register_device(struct virtio_device *vdev, int vq_num, struct virtqueue **vqs); + +/** + * @brief Setup a virtqueue structure. + * + * @param dev Pointer to device structure. + * @param idx Index of the virtqueue. + * @param vq Pointer to virtqueue structure. + * @param cb Pointer to virtqueue callback. Can be NULL. + * @param cb_arg Argument for the virtqueue callback. + * + * @return pointer to virtqueue structure. + */ + +struct virtqueue *virtio_mmio_setup_virtqueue(struct virtio_device *vdev, + unsigned int idx, + struct virtqueue *vq, + void (*cb)(void *), + void *cb_arg, + const char *vq_name); + +/** + * @brief VIRTIO MMIO device initialization. + * + * @param vmdev Pointer to virtio_mmio_device structure. + * @param virt_mem_ptr Guest virtio (shared) memory base address (virtual). + * @param cfg_mem_ptr Virtio device configuration memory base address (virtual). + * @param user_data Pointer to custom user data. + * + * @return int 0 for success. + */ + +int virtio_mmio_device_init(struct virtio_mmio_device *vmdev, uintptr_t virt_mem_ptr, + uintptr_t cfg_mem_ptr, void *user_data); + +/** + * @brief VIRTIO MMIO interrupt service routine. + * + * @param vdev Pointer to virtio_device structure. + */ + +void virtio_mmio_isr(struct virtio_device *vdev); + +#ifdef __cplusplus +} +#endif + +#endif /* OPENAMP_VIRTIO_MMIO_H */ diff --git a/lib/include/openamp/virtqueue.h b/lib/include/openamp/virtqueue.h index 237f9be81..24a0e5b7b 100644 --- a/lib/include/openamp/virtqueue.h +++ b/lib/include/openamp/virtqueue.h @@ -63,6 +63,7 @@ struct virtqueue { uint16_t vq_queue_index; uint16_t vq_nentries; void (*callback)(struct virtqueue *vq); + void *priv; void (*notify)(struct virtqueue *vq); struct vring vq_ring; uint16_t vq_free_cnt; @@ -334,6 +335,31 @@ uint32_t virtqueue_get_desc_size(struct virtqueue *vq); uint32_t virtqueue_get_buffer_length(struct virtqueue *vq, uint16_t idx); void *virtqueue_get_buffer_addr(struct virtqueue *vq, uint16_t idx); +/** + * @brief Test if virtqueue is empty + * + * @param vq Pointer to VirtIO queue control block + * + * @return 1 if virtqueue is empty + */ +static inline int virtqueue_empty(struct virtqueue *vq) +{ + return (vq->vq_nentries == vq->vq_free_cnt); +} + +/** + * @brief Test if virtqueue is full + * + * @param vq Pointer to VirtIO queue control block + * + * @return 1 if virtqueue is full + */ + +static inline int virtqueue_full(struct virtqueue *vq) +{ + return (vq->vq_free_cnt == 0); +} + #if defined __cplusplus } #endif diff --git a/lib/virtio/virtio.c b/lib/virtio/virtio.c index c745426fc..6083e4f00 100644 --- a/lib/virtio/virtio.c +++ b/lib/virtio/virtio.c @@ -96,7 +96,7 @@ void virtio_describe(struct virtio_device *dev, const char *msg, int virtio_create_virtqueues(struct virtio_device *vdev, unsigned int flags, unsigned int nvqs, const char *names[], - vq_callback callbacks[]) + vq_callback callbacks[], void *callback_args[]) { struct virtio_vring_info *vring_info; struct vring_alloc_info *vring_alloc; @@ -109,7 +109,7 @@ int virtio_create_virtqueues(struct virtio_device *vdev, unsigned int flags, if (vdev->func && vdev->func->create_virtqueues) { return vdev->func->create_virtqueues(vdev, flags, nvqs, - names, callbacks); + names, callbacks, callback_args); } num_vrings = vdev->vrings_num; diff --git a/lib/virtio_mmio/CMakeLists.txt b/lib/virtio_mmio/CMakeLists.txt new file mode 100644 index 000000000..f25d00c66 --- /dev/null +++ b/lib/virtio_mmio/CMakeLists.txt @@ -0,0 +1,3 @@ +if (WITH_VIRTIO_MMIO_DRV) +collect (PROJECT_LIB_SOURCES virtio_mmio_drv.c) +endif (WITH_VIRTIO_MMIO_DRV) diff --git a/lib/virtio_mmio/virtio_mmio_drv.c b/lib/virtio_mmio/virtio_mmio_drv.c new file mode 100644 index 000000000..09193c78c --- /dev/null +++ b/lib/virtio_mmio/virtio_mmio_drv.c @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2022 Wind River Systems, Inc. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include + +void virtio_mmio_isr(struct virtio_device *vdev); + +typedef void (*virtio_mmio_vq_callback)(void *); + +static int virtio_mmio_create_virtqueues(struct virtio_device *vdev, unsigned int flags, + unsigned int nvqs, const char *names[], + vq_callback callbacks[], void *callback_args[]); + +static inline void virtio_mmio_write32(struct virtio_device *vdev, int offset, uint32_t value) +{ + struct virtio_mmio_device *vmdev = metal_container_of(vdev, + struct virtio_mmio_device, vdev); + + metal_io_write32(vmdev->cfg_io, offset, value); +} + +static inline uint32_t virtio_mmio_read32(struct virtio_device *vdev, int offset) +{ + struct virtio_mmio_device *vmdev = metal_container_of(vdev, + struct virtio_mmio_device, vdev); + + return metal_io_read32(vmdev->cfg_io, offset); +} + +static inline uint8_t virtio_mmio_read8(struct virtio_device *vdev, int offset) +{ + struct virtio_mmio_device *vmdev = metal_container_of(vdev, + struct virtio_mmio_device, vdev); + + return metal_io_read8(vmdev->cfg_io, offset); +} + +static inline void virtio_mmio_set_status(struct virtio_device *vdev, uint8_t status) +{ + virtio_mmio_write32(vdev, VIRTIO_MMIO_STATUS, status); +} + +static uint8_t virtio_mmio_get_status(struct virtio_device *vdev) +{ + return virtio_mmio_read32(vdev, VIRTIO_MMIO_STATUS); +} + +static void virtio_mmio_write_config(struct virtio_device *vdev, + uint32_t offset, void *dst, int length) +{ + (void)(vdev); + (void)(offset); + (void)(dst); + (void)length; + + metal_log(METAL_LOG_WARNING, "%s not supported\n", __func__); +} + +static void virtio_mmio_read_config(struct virtio_device *vdev, + uint32_t offset, void *dst, int length) +{ + int i; + uint8_t *d = dst; + (void)(offset); + + for (i = 0; i < length; i++) + d[i] = virtio_mmio_read8(vdev, VIRTIO_MMIO_CONFIG + i); +} + +static uint32_t _virtio_mmio_get_features(struct virtio_device *vdev, int idx) +{ + uint32_t hfeatures; + + /* Writing selection register VIRTIO_MMIO_DEVICE_FEATURES_SEL. In pure AMP + * mode this needs to be followed by a synchonization w/ the device + * before reading VIRTIO_MMIO_DEVICE_FEATURES + */ + virtio_mmio_write32(vdev, VIRTIO_MMIO_DEVICE_FEATURES_SEL, idx); + hfeatures = virtio_mmio_read32(vdev, VIRTIO_MMIO_DEVICE_FEATURES); + return hfeatures & vdev->features; +} + +static uint32_t virtio_mmio_get_features(struct virtio_device *vdev) +{ + return _virtio_mmio_get_features(vdev, 0); +} + +/* This is more like negotiate_features */ +static void _virtio_mmio_set_features(struct virtio_device *vdev, + uint32_t features, int idx) +{ + uint32_t hfeatures; + + /* Writing selection register VIRTIO_MMIO_DEVICE_FEATURES_SEL. In pure AMP + * mode this needs to be followed by a synchonization w/ the device + * before reading VIRTIO_MMIO_DEVICE_FEATURES + */ + virtio_mmio_write32(vdev, VIRTIO_MMIO_DEVICE_FEATURES_SEL, idx); + hfeatures = virtio_mmio_read32(vdev, VIRTIO_MMIO_DEVICE_FEATURES); + features &= hfeatures; + virtio_mmio_write32(vdev, VIRTIO_MMIO_DRIVER_FEATURES, features); + vdev->features = features; +} + +static void virtio_mmio_set_features(struct virtio_device *vdev, uint32_t features) +{ + _virtio_mmio_set_features(vdev, features, 0); +} + +static void virtio_mmio_reset_device(struct virtio_device *vdev) +{ + virtio_mmio_set_status(vdev, 0); +} + +static void virtio_mmio_notify(struct virtqueue *vq) +{ + /* VIRTIO_F_NOTIFICATION_DATA is not supported for now */ + virtio_mmio_write32(vq->vq_dev, VIRTIO_MMIO_QUEUE_NOTIFY, vq->vq_queue_index); +} + +const struct virtio_dispatch virtio_mmio_dispatch = { + .create_virtqueues = virtio_mmio_create_virtqueues, + .get_status = virtio_mmio_get_status, + .set_status = virtio_mmio_set_status, + .get_features = virtio_mmio_get_features, + .set_features = virtio_mmio_set_features, + .read_config = virtio_mmio_read_config, + .write_config = virtio_mmio_write_config, + .reset_device = virtio_mmio_reset_device, + .notify = virtio_mmio_notify, +}; + +static int virtio_mmio_get_metal_io(struct virtio_device *vdev, uintptr_t virt_mem_ptr, + uintptr_t cfg_mem_ptr) +{ + struct metal_init_params metal_params = METAL_INIT_DEFAULTS; + struct metal_device *device; + int32_t err; + int device_idx = 0; + struct virtio_mmio_device *vmdev = metal_container_of(vdev, + struct virtio_mmio_device, vdev); + + metal_params.log_level = METAL_LOG_DEBUG; + + /* Setup shared memory device */ + vmdev->shm_device.regions[0].physmap = (metal_phys_addr_t *)&vmdev->shm_mem.base; + vmdev->shm_device.regions[0].virt = (void *)virt_mem_ptr; + vmdev->shm_device.regions[0].size = vmdev->shm_mem.size; + + VIRTIO_ASSERT((METAL_MAX_DEVICE_REGIONS > 1), + "METAL_MAX_DEVICE_REGIONS must be greater that 1"); + + vmdev->shm_device.regions[1].physmap = (metal_phys_addr_t *)&vmdev->cfg_mem.base; + vmdev->shm_device.regions[1].virt = (void *)cfg_mem_ptr; + vmdev->shm_device.regions[1].size = vmdev->cfg_mem.size; + + /* Libmetal setup */ + err = metal_init(&metal_params); + if (err) { + metal_log(METAL_LOG_ERROR, "metal_init: failed - error code %d\n", err); + return err; + } + + err = metal_register_generic_device(&vmdev->shm_device); + if (err) { + metal_log(METAL_LOG_ERROR, "Couldn't register shared memory device: %d\n", err); + return err; + } + + err = metal_device_open("generic", vmdev->shm_device.name, &device); + if (err) { + metal_log(METAL_LOG_ERROR, "metal_device_open failed: %d", err); + return err; + } + + vmdev->shm_io = metal_device_io_region(device, device_idx); + if (!vmdev->shm_io) { + metal_log(METAL_LOG_ERROR, "metal_device_io_region failed to get region %d", + device_idx); + return err; + } + + vmdev->cfg_io = metal_device_io_region(device, 1); + if (!vmdev->cfg_io) { + metal_log(METAL_LOG_ERROR, "metal_device_io_region failed to get region 1"); + return err; + } + + return 0; +} + +uint32_t virtio_mmio_get_max_elem(struct virtio_device *vdev, int idx) +{ + /* Select the queue we're interested in by writing selection register + * VIRTIO_MMIO_QUEUE_SEL. In pure AMP mode this needs to be followed by a + * synchonization w/ the device before reading VIRTIO_MMIO_QUEUE_NUM_MAX + */ + virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_SEL, idx); + return virtio_mmio_read32(vdev, VIRTIO_MMIO_QUEUE_NUM_MAX); +} + +int virtio_mmio_device_init(struct virtio_mmio_device *vmdev, uintptr_t virt_mem_ptr, + uintptr_t cfg_mem_ptr, void *user_data) +{ + struct virtio_device *vdev = &vmdev->vdev; + uint32_t magic, version, devid, vendor; + + vdev->role = vmdev->device_mode; + vdev->priv = vmdev; + vdev->func = &virtio_mmio_dispatch; + vmdev->user_data = user_data; + + /* Set metal io mem ops */ + virtio_mmio_get_metal_io(vdev, virt_mem_ptr, cfg_mem_ptr); + + magic = virtio_mmio_read32(vdev, VIRTIO_MMIO_MAGIC_VALUE); + if (magic != VIRTIO_MMIO_MAGIC_VALUE_STRING) { + metal_log(METAL_LOG_ERROR, "Bad magic value %08x\n", magic); + return -1; + } + + version = virtio_mmio_read32(vdev, VIRTIO_MMIO_VERSION); + devid = virtio_mmio_read32(vdev, VIRTIO_MMIO_DEVICE_ID); + if (devid == 0) { + /* Placeholder */ + return -1; + } + + if (version != 1) { + metal_log(METAL_LOG_ERROR, "Bad version %08x\n", version); + return -1; + } + + vendor = virtio_mmio_read32(vdev, VIRTIO_MMIO_VENDOR_ID); + metal_log(METAL_LOG_DEBUG, "VIRTIO %08x:%08x\n", vendor, devid); + + vdev->id.version = version; + vdev->id.device = devid; + vdev->id.vendor = vendor; + + virtio_mmio_set_status(vdev, VIRTIO_CONFIG_STATUS_ACK); + virtio_mmio_write32(vdev, VIRTIO_MMIO_GUEST_PAGE_SIZE, 4096); + + return 0; +} + +/* Register preallocated virtqueues */ +void virtio_mmio_register_device(struct virtio_device *vdev, int vq_num, struct virtqueue **vqs) +{ + int i; + + vdev->vrings_info = metal_allocate_memory(sizeof(struct virtio_vring_info) * vq_num); + /* TODO: handle error case */ + for (i = 0; i < vq_num; i++) { + vdev->vrings_info[i].vq = vqs[i]; + } + vdev->vrings_num = vq_num; +} + +struct virtqueue *virtio_mmio_setup_virtqueue(struct virtio_device *vdev, + unsigned int idx, + struct virtqueue *vq, + void (*cb)(void *), + void *cb_arg, + const char *vq_name) +{ + uint32_t maxq; + struct virtio_vring_info _vring_info = {0}; + struct virtio_vring_info *vring_info = &_vring_info; + struct vring_alloc_info *vring_alloc_info; + struct virtio_mmio_device *vmdev = metal_container_of(vdev, + struct virtio_mmio_device, vdev); + + if (vdev->role != (unsigned int)VIRTIO_DEV_DRIVER) { + metal_log(METAL_LOG_ERROR, "Only VIRTIO_DEV_DRIVER is currently supported\n"); + return NULL; + } + + if (!vq) { + metal_log(METAL_LOG_ERROR, + "Only preallocated virtqueues are currently supported\n"); + return NULL; + } + + if (vdev->id.version != 0x1) { + metal_log(METAL_LOG_ERROR, + "Only VIRTIO MMIO version 1 is currently supported\n"); + return NULL; + } + + vring_info->io = vmdev->shm_io; + vring_info->info.num_descs = virtio_mmio_get_max_elem(vdev, idx); + vring_info->info.align = VRING_ALIGNMENT; + + /* Preallocated virtqueues; vrings can be already declared via VIRTQUEUE_DECLARE */ + /* Check if vrings are already configured */ + if (vq->vq_nentries != 0 && vq->vq_nentries == vq->vq_free_cnt && + vq->vq_ring.desc) { + vring_info->info.vaddr = vq->vq_ring.desc; + vring_info->vq = vq; + } + vring_info->info.num_descs = vq->vq_nentries; + + vq->vq_dev = vdev; + + vring_alloc_info = &vring_info->info; + + unsigned int role_bk = vdev->role; + /* Assign OA VIRTIO_DEV_DRIVER role to allow virtio guests to setup the vrings */ + vdev->role = (unsigned int)VIRTIO_DEV_DRIVER; + if (virtqueue_create(vdev, idx, vq_name, vring_alloc_info, (void (*)(struct virtqueue *))cb, + vdev->func->notify, vring_info->vq)) { + metal_log(METAL_LOG_ERROR, "virtqueue_create failed\n"); + return NULL; + } + vdev->role = role_bk; + vq->priv = cb_arg; + virtqueue_set_shmem_io(vq, vmdev->shm_io); + + /* Writing selection register VIRTIO_MMIO_QUEUE_SEL. In pure AMP + * mode this needs to be followed by a synchonization w/ the device + * before reading VIRTIO_MMIO_QUEUE_NUM_MAX + */ + virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_SEL, idx); + maxq = virtio_mmio_read32(vdev, VIRTIO_MMIO_QUEUE_NUM_MAX); + VIRTIO_ASSERT((maxq != 0), + "VIRTIO_MMIO_QUEUE_NUM_MAX cannot be 0"); + VIRTIO_ASSERT((maxq >= vq->vq_nentries), + "VIRTIO_MMIO_QUEUE_NUM_MAX must be greater than vqueue->vq_nentries"); + virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_NUM, vq->vq_nentries); + virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_ALIGN, 4096); + virtio_mmio_write32(vdev, VIRTIO_MMIO_QUEUE_PFN, + ((uintptr_t)metal_io_virt_to_phys(vq->shm_io, + (char *)vq->vq_ring.desc)) / 4096); + + vdev->vrings_info[vdev->vrings_num].vq = vq; + vdev->vrings_num++; + virtqueue_enable_cb(vq); + + return vq; +} + +void virtio_mmio_isr(struct virtio_device *vdev) +{ + struct virtio_vring_info *vrings_info = vdev->vrings_info; + + uint32_t isr = virtio_mmio_read32(vdev, VIRTIO_MMIO_INTERRUPT_STATUS); + struct virtqueue *vq; + unsigned int i; + + if (isr & VIRTIO_MMIO_INT_VRING) { + for (i = 0; i < vdev->vrings_num; i++) { + vq = vrings_info[i].vq; + if (vq->callback) + vq->callback(vq->priv); + } + } + + if (isr & ~(VIRTIO_MMIO_INT_VRING)) + metal_log(METAL_LOG_WARNING, "Unhandled interrupt type: 0x%x\n", isr); + + virtio_mmio_write32(vdev, VIRTIO_MMIO_INTERRUPT_ACK, isr); +} + +static int virtio_mmio_create_virtqueues(struct virtio_device *vdev, unsigned int flags, + unsigned int nvqs, const char *names[], + vq_callback callbacks[], void *callback_args[]) +{ + struct virtqueue *vq = NULL; + struct virtqueue *vring_vq = NULL; + void (*cb)(void *) = NULL; + void *cb_arg = NULL; + unsigned int i; + + (void)flags; + + if (!vdev || !names || !vdev->vrings_info) + return -EINVAL; + + for (i = 0; i < nvqs; i++) { + vring_vq = NULL; + cb = NULL; + cb_arg = NULL; + if (vdev->vrings_info[i].vq) + vring_vq = vdev->vrings_info[i].vq; + if (callbacks) + cb = (virtio_mmio_vq_callback)callbacks[i]; + if (callback_args) + cb_arg = callback_args[i]; + vq = virtio_mmio_setup_virtqueue(vdev, i, vring_vq, cb, cb_arg, names[i]); + if (!vq) + return -ENODEV; + } + + return 0; +}