Merge pull request #1436 from selvintxavier/code_refactor

bnxt_re/lib: Code refactoring in the resource allocation verbs

providers/bnxt_re/bnxt_re-abi.h

@@ -239,6 +239,8 @@ enum bnxt_re_db_epoch_flag_shift {
 	BNXT_RE_DB_EPOCH_HEAD_SHIFT	= (BNXT_RE_DB_EPOCH_SHIFT - 1)
 };
 
+#define BNXT_RE_STATIC_WQE_MAX_SGE              0x06
+
 enum bnxt_re_modes {
 	BNXT_RE_WQE_MODE_STATIC =	0x00,
 	BNXT_RE_WQE_MODE_VARIABLE =	0x01

providers/bnxt_re/db.c

@@ -171,8 +171,8 @@ void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq)
 	uint32_t epoch;
 
 	bnxt_re_do_pacing(cq->cntx, &cq->rand);
-	epoch = (cq->cqq.flags & BNXT_RE_FLAG_EPOCH_HEAD_MASK) << BNXT_RE_DB_EPOCH_HEAD_SHIFT;
-	bnxt_re_init_db_hdr(&hdr, cq->cqq.head | epoch, cq->cqid, 0, BNXT_RE_QUE_TYPE_CQ);
+	epoch = (cq->cqq->flags & BNXT_RE_FLAG_EPOCH_HEAD_MASK) << BNXT_RE_DB_EPOCH_HEAD_SHIFT;
+	bnxt_re_init_db_hdr(&hdr, cq->cqq->head | epoch, cq->cqid, 0, BNXT_RE_QUE_TYPE_CQ);
 	bnxt_re_ring_db(cq->udpi, &hdr);
 }
 
@@ -191,8 +191,8 @@ void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag)
 	}
 
 	bnxt_re_do_pacing(cq->cntx, &cq->rand);
-	epoch = (cq->cqq.flags & BNXT_RE_FLAG_EPOCH_HEAD_MASK) <<  BNXT_RE_DB_EPOCH_HEAD_SHIFT;
-	bnxt_re_init_db_hdr(&hdr, cq->cqq.head | epoch, cq->cqid, toggle, aflag);
+	epoch = (cq->cqq->flags & BNXT_RE_FLAG_EPOCH_HEAD_MASK) <<  BNXT_RE_DB_EPOCH_HEAD_SHIFT;
+	bnxt_re_init_db_hdr(&hdr, cq->cqq->head | epoch, cq->cqid, toggle, aflag);
 	bnxt_re_ring_db(cq->udpi, &hdr);
 }
 

providers/bnxt_re/main.h

@@ -93,9 +93,10 @@ struct bnxt_re_cq {
 	struct ibv_cq ibvcq;
 	uint32_t cqid;
 	struct bnxt_re_context *cntx;
-	struct bnxt_re_queue cqq;
-	struct bnxt_re_queue resize_cqq;
+	struct bnxt_re_queue *cqq;
 	struct bnxt_re_dpi *udpi;
+	struct bnxt_re_mem *mem;
+	struct bnxt_re_mem *resize_mem;
 	struct list_head sfhead;
 	struct list_head rfhead;
 	struct list_head prev_cq_head;
@@ -167,13 +168,15 @@ struct bnxt_re_srq {
 	struct bnxt_re_wrid *srwrid;
 	struct bnxt_re_dpi *udpi;
 	struct xorshift32_state rand;
+	struct bnxt_re_mem *mem;
 	uint32_t srqid;
 	int start_idx;
 	int last_idx;
 	bool arm_req;
 };
 
 struct bnxt_re_joint_queue {
+	struct bnxt_re_context *cntx;
 	struct bnxt_re_queue *hwque;
 	struct bnxt_re_wrid *swque;
 	uint32_t start_idx;
@@ -205,7 +208,7 @@ struct bnxt_re_qp {
 	uint8_t push_st_en;
 	uint16_t max_push_sz;
 	uint8_t qptyp;
-	/* irdord? */
+	struct bnxt_re_mem *mem;
 };
 
 struct bnxt_re_mr {
@@ -530,7 +533,7 @@ static inline uint8_t bnxt_re_is_cqe_valid(struct bnxt_re_cq *cq,
 
 static inline void bnxt_re_change_cq_phase(struct bnxt_re_cq *cq)
 {
-	if (!cq->cqq.head)
+	if (!cq->cqq->head)
 		cq->phase = (~cq->phase & BNXT_RE_BCQE_PH_MASK);
 }
 

providers/bnxt_re/memory.c

@@ -38,40 +38,69 @@
  */
 
 #include <string.h>
+#include <malloc.h>
 #include <sys/mman.h>
 #include <util/util.h>
 
 #include "main.h"
 
-int bnxt_re_alloc_aligned(struct bnxt_re_queue *que, uint32_t pg_size)
+void bnxt_re_free_mem(struct bnxt_re_mem *mem)
 {
-	int ret, bytes;
-
-	bytes = (que->depth * que->stride);
-	que->bytes = align(bytes, pg_size);
-	que->va = mmap(NULL, que->bytes, PROT_READ | PROT_WRITE,
-		       MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-	if (que->va == MAP_FAILED) {
-		que->bytes = 0;
-		return errno;
+	if (mem->va_head) {
+		ibv_dofork_range(mem->va_head, mem->size);
+		munmap(mem->va_head, mem->size);
 	}
-	/* Touch pages before proceeding. */
-	memset(que->va, 0, que->bytes);
 
-	ret = ibv_dontfork_range(que->va, que->bytes);
-	if (ret) {
-		munmap(que->va, que->bytes);
-		que->bytes = 0;
-	}
+	free(mem);
+}
+
+void *bnxt_re_alloc_mem(size_t size, uint32_t pg_size)
+{
+	struct bnxt_re_mem *mem;
+
+	mem = calloc(1, sizeof(*mem));
+	if (!mem)
+		return NULL;
+
+	size = align(size, pg_size);
+	mem->size = size;
+	mem->va_head = mmap(NULL, size, PROT_READ | PROT_WRITE,
+			    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (mem->va_head == MAP_FAILED)
+		goto bail;
 
-	return ret;
+	if (ibv_dontfork_range(mem->va_head, size))
+		goto unmap;
+
+	mem->head = 0;
+	mem->tail = 0;
+	mem->va_tail = (void *)((char *)mem->va_head + size);
+	return mem;
+unmap:
+	munmap(mem->va_head, size);
+bail:
+	free(mem);
+	return NULL;
 }
 
-void bnxt_re_free_aligned(struct bnxt_re_queue *que)
+void *bnxt_re_get_obj(struct bnxt_re_mem *mem, size_t req)
 {
-	if (que->bytes) {
-		ibv_dofork_range(que->va, que->bytes);
-		munmap(que->va, que->bytes);
-		que->bytes = 0;
-	}
+	void *va;
+
+	if ((mem->size - mem->tail - req) < mem->head)
+		return NULL;
+	mem->tail += req;
+	va = (void *)((char *)mem->va_tail - mem->tail);
+	return va;
+}
+
+void *bnxt_re_get_ring(struct bnxt_re_mem *mem, size_t req)
+{
+	void *va;
+
+	if ((mem->head + req) > (mem->size - mem->tail))
+		return NULL;
+	va = (void *)((char *)mem->va_head + mem->head);
+	mem->head += req;
+	return va;
 }

providers/bnxt_re/memory.h

@@ -41,6 +41,27 @@
 #define __MEMORY_H__
 
 #include <pthread.h>
+#include "main.h"
+
+struct bnxt_re_mem {
+	void *va_head;
+	void *va_tail;
+	uint32_t head;
+	uint32_t tail;
+	uint32_t size;
+	uint32_t pad;
+};
+
+#define BNXT_RE_QATTR_SQ_INDX	0
+#define BNXT_RE_QATTR_RQ_INDX	1
+struct bnxt_re_qattr {
+	uint32_t esize;
+	uint32_t slots;
+	uint32_t nwr;
+	uint32_t sz_ring;
+	uint32_t sz_shad;
+	uint32_t sw_nwr;
+};
 
 struct bnxt_re_queue {
 	void *va;
@@ -68,8 +89,6 @@ struct bnxt_re_queue {
 	uint32_t msn_tbl_sz;
 };
 
-int bnxt_re_alloc_aligned(struct bnxt_re_queue *que, uint32_t pg_size);
-void bnxt_re_free_aligned(struct bnxt_re_queue *que);
 
 /* Basic queue operation */
 static inline void *bnxt_re_get_hwqe(struct bnxt_re_queue *que, uint32_t idx)
@@ -123,4 +142,9 @@ static inline void bnxt_re_incr_head(struct bnxt_re_queue *que, uint8_t cnt)
 	}
 }
 
+void bnxt_re_free_mem(struct bnxt_re_mem *mem);
+void *bnxt_re_alloc_mem(size_t size, uint32_t pg_size);
+void *bnxt_re_get_obj(struct bnxt_re_mem *mem, size_t req);
+void *bnxt_re_get_ring(struct bnxt_re_mem *mem, size_t req);
+
 #endif