vdpa/virtio: move iommu domain lock to driver private structure

Move iommu domain lock to virtio_vdpa_priv so that it could protect
iommu domain when vhost thread and RPC thread access it in the meantime.

Signed-off-by: Chenbo Xia <chenbox@nvidia.com>

drivers/vdpa/virtio/virtio_vdpa.c

@@ -117,7 +117,6 @@ virtio_vdpa_find_iommu_domain_by_uuid(const rte_uuid_t vm_uuid)
 		iommu_domain->vfio_container_fd = -1;
 		iommu_domain->container_ref_cnt = 0;
 		iommu_domain->mem_tbl_ref_cnt = 0;
-		pthread_mutex_init(&iommu_domain->domain_lock, NULL);
 		TAILQ_INSERT_TAIL(&virtio_iommu_domain_list, iommu_domain, next);
 	}
 
@@ -809,8 +808,10 @@ virtio_vdpa_dev_cleanup(int vid)
 	if (ret < 0)
 		DRV_LOG(ERR, "Failed to remove mem table: %s", vdev->device->name);
 
+	pthread_mutex_lock(&priv->domain_lock);
 	iommu_domain = priv->iommu_domain;
-	pthread_mutex_lock(&iommu_domain->domain_lock);
+	if (iommu_domain == NULL)
+		goto unlock;
 	if (priv->mem_tbl_set) {
 		iommu_domain->mem_tbl_ref_cnt--;
 		priv->mem_tbl_set = false;
@@ -829,7 +830,8 @@ virtio_vdpa_dev_cleanup(int vid)
 err:
 		iommu_domain->mem.nregions = 0;
 	}
-	pthread_mutex_unlock(&iommu_domain->domain_lock);			
+unlock:
+	pthread_mutex_unlock(&priv->domain_lock);			
 
 	return 0;
 }
@@ -939,8 +941,10 @@ virtio_vdpa_dev_set_mem_table(int vid)
 		}
 	}
 
+	pthread_mutex_lock(&priv->domain_lock);
 	iommu_domain = priv->iommu_domain;
-	pthread_mutex_lock(&iommu_domain->domain_lock);
+	if (iommu_domain == NULL)
+		goto err;
 	/* Unmap region does not exist in current */
 	for (i = 0; i < iommu_domain->mem.nregions; i++) {
 		reg = &iommu_domain->mem.regions[i];
@@ -1020,7 +1024,7 @@ virtio_vdpa_dev_set_mem_table(int vid)
 	virtio_vdpa_dev_store_mem_tbl(priv);
 
 err:
-	pthread_mutex_unlock(&iommu_domain->domain_lock);
+	pthread_mutex_unlock(&priv->domain_lock);
 	return ret;
 }
 
@@ -2061,8 +2065,8 @@ virtio_vdpa_dev_do_remove(struct rte_pci_device *pci_dev, struct virtio_vdpa_pri
 		virtio_pci_dev_free(priv->vpdev);
 	}
 
+	pthread_mutex_lock(&priv->domain_lock);
 	if (iommu_domain) {
-		pthread_mutex_lock(&iommu_domain->domain_lock);
 		iommu_domain->container_ref_cnt--;
 		if (iommu_domain->container_ref_cnt == 0) {
 			if (priv->vfio_container_fd >= 0) {
@@ -2072,9 +2076,10 @@ virtio_vdpa_dev_do_remove(struct rte_pci_device *pci_dev, struct virtio_vdpa_pri
 			if (!rte_uuid_is_null(iommu_domain->vm_uuid))
 				TAILQ_REMOVE(&virtio_iommu_domain_list, iommu_domain, next);
 			rte_free(iommu_domain);
-		}	
-		pthread_mutex_unlock(&iommu_domain->domain_lock);
+			priv->iommu_domain = NULL;
+		}
 	}
+	pthread_mutex_unlock(&priv->domain_lock);
 
 	if (priv->state_mz)
 		rte_memzone_free(priv->state_mz);
@@ -2210,7 +2215,6 @@ virtio_vdpa_dev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		iommu_domain->vfio_container_fd = -1;
 		iommu_domain->container_ref_cnt = 0;
 		iommu_domain->mem_tbl_ref_cnt = 0;
-		pthread_mutex_init(&iommu_domain->domain_lock, NULL);
 	}
 
 	if (iommu_domain->container_ref_cnt == RTE_MAX_VFIO_GROUPS) {
@@ -2220,6 +2224,7 @@ virtio_vdpa_dev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	}
 
 	priv->iommu_domain = iommu_domain;
+	pthread_mutex_init(&priv->domain_lock, NULL);
 
 	if (!strcmp(cached_ctx.vf_name.dev_bdf, devname)) {
 		priv->restore = true;

drivers/vdpa/virtio/virtio_vdpa.h

@@ -34,7 +34,6 @@ struct virtio_vdpa_iommu_domain {
 	struct virtio_vdpa_vf_drv_mem mem;
 	int container_ref_cnt;
 	int mem_tbl_ref_cnt;
-	pthread_mutex_t domain_lock;
 };
 
 struct virtio_vdpa_vring_info {
@@ -64,6 +63,7 @@ struct virtio_vdpa_priv {
 	const struct rte_memzone *state_mz_remote; /* This is used get state frome contoller */
 	const struct virtio_vdpa_device_callback *dev_ops;
 	struct virtio_vdpa_iommu_domain *iommu_domain;
+	pthread_mutex_t domain_lock;
 	enum virtio_internal_status lm_status;
 	int state_size;
 	int vfio_container_fd;