efa: Fix CQ doorbell unmap on CQ destroy

[ Upstream commit 515a351 ]

Remove the CQ doorbell offset on munmap to get the aligned address
provided by mmap.
When using the unaligned address IB core gets wrong VMA to its umap
close callback with private data NULL so it doesn't drop the reference
on the mmap entry. This means that we don't remove the entry from the
mmap xarray and when requesting to close the device it fails because of
open resources on it.

Fixes: 94ed75b ("efa: CQ notifications support")
Reviewed-by: Daniel Kranzdorf <dkkranzd@amazon.com>
Reviewed-by: Firas Jahjah <firasj@amazon.com>
Signed-off-by: Yonatan Nachum <ynachum@amazon.com>
Signed-off-by: Nicolas Morey <nmorey@suse.com>

providers/efa/efa.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2019-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef __EFA_H__
@@ -70,6 +70,7 @@ struct efa_cq {
 	uint8_t *buf;
 	size_t buf_size;
 	uint32_t *db;
+	uint8_t *db_mmap_addr;
 	uint16_t cc; /* Consumer Counter */
 	uint8_t cmd_sn;
 	uint16_t num_sub_cqs;

providers/efa/verbs.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2019-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2019-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <assert.h>
@@ -868,13 +868,13 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	}
 
 	if (resp.comp_mask & EFA_CREATE_CQ_RESP_DB_OFF) {
-		cq->db = mmap(NULL,
-			      to_efa_dev(ibvctx->device)->pg_sz, PROT_WRITE,
-			      MAP_SHARED, ibvctx->cmd_fd, resp.db_mmap_key);
-		if (cq->db == MAP_FAILED)
+		cq->db_mmap_addr = mmap(NULL,
+					to_efa_dev(ibvctx->device)->pg_sz, PROT_WRITE,
+					MAP_SHARED, ibvctx->cmd_fd, resp.db_mmap_key);
+		if (cq->db_mmap_addr == MAP_FAILED)
 			goto err_unmap_cq;
 
-		cq->db = (uint32_t *)((uint8_t *)cq->db + resp.db_off);
+		cq->db = (uint32_t *)(cq->db_mmap_addr + resp.db_off);
 	}
 
 	efa_cq_fill_pfns(cq, attr, efa_attr);
@@ -966,7 +966,7 @@ int efa_destroy_cq(struct ibv_cq *ibvcq)
 		return err;
 	}
 
-	munmap(cq->db, to_efa_dev(cq->verbs_cq.cq.context->device)->pg_sz);
+	munmap(cq->db_mmap_addr, to_efa_dev(cq->verbs_cq.cq.context->device)->pg_sz);
 	munmap(cq->buf, cq->buf_size);
 
 	pthread_spin_destroy(&cq->lock);