Skip to content

Commit

Permalink
libhns: Add support for lock-free CQ
Browse files Browse the repository at this point in the history 
Drop CQ locks when associated to a PAD holding a TD.

Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
  • Loading branch information
Junxian Huang authored and Junxian Huang committed Jul 5, 2024
1 parent ae35032 commit 8c865c3
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 28 deletions.
3 changes: 2 additions & 1 deletion providers/hns/hns_roce_u.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ struct hns_roce_pad {
struct hns_roce_cq {
struct verbs_cq verbs_cq;
struct hns_roce_buf buf;
pthread_spinlock_t lock;
struct hns_roce_spinlock hr_lock;
unsigned int cqn;
unsigned int cq_depth;
unsigned int cons_index;
Expand All @@ -265,6 +265,7 @@ struct hns_roce_cq {
unsigned long flags;
unsigned int cqe_size;
struct hns_roce_v2_cqe *cqe;
struct ibv_pd *parent_domain;
};

struct hns_roce_idx_que {
Expand Down
46 changes: 23 additions & 23 deletions providers/hns/hns_roce_u_hw_v2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,9 +268,9 @@ static int hns_roce_v2_wq_overflow(struct hns_roce_wq *wq, unsigned int nreq,
if (cur + nreq < wq->max_post)
return 0;

pthread_spin_lock(&cq->lock);
hns_roce_spin_lock(&cq->hr_lock);
cur = wq->head - wq->tail;
pthread_spin_unlock(&cq->lock);
hns_roce_spin_unlock(&cq->hr_lock);

return cur + nreq >= wq->max_post;
}
Expand Down Expand Up @@ -737,7 +737,7 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne,
int err = V2_CQ_OK;
int npolled;

pthread_spin_lock(&cq->lock);
hns_roce_spin_lock(&cq->hr_lock);

for (npolled = 0; npolled < ne; ++npolled) {
err = hns_roce_poll_one(ctx, &qp, cq, wc + npolled);
Expand All @@ -752,7 +752,7 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne,
update_cq_db(ctx, cq);
}

pthread_spin_unlock(&cq->lock);
hns_roce_spin_unlock(&cq->hr_lock);

return err == V2_CQ_POLL_ERR ? err : npolled;
}
Expand Down Expand Up @@ -1523,9 +1523,9 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
static void hns_roce_v2_cq_clean(struct hns_roce_cq *cq, unsigned int qpn,
struct hns_roce_srq *srq)
{
pthread_spin_lock(&cq->lock);
hns_roce_spin_lock(&cq->hr_lock);
__hns_roce_v2_cq_clean(cq, qpn, srq);
pthread_spin_unlock(&cq->lock);
hns_roce_spin_unlock(&cq->hr_lock);
}

static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr,
Expand Down Expand Up @@ -1611,18 +1611,18 @@ static void hns_roce_lock_cqs(struct ibv_qp *qp)

if (send_cq && recv_cq) {
if (send_cq == recv_cq) {
pthread_spin_lock(&send_cq->lock);
hns_roce_spin_lock(&send_cq->hr_lock);
} else if (send_cq->cqn < recv_cq->cqn) {
pthread_spin_lock(&send_cq->lock);
pthread_spin_lock(&recv_cq->lock);
hns_roce_spin_lock(&send_cq->hr_lock);
hns_roce_spin_lock(&recv_cq->hr_lock);
} else {
pthread_spin_lock(&recv_cq->lock);
pthread_spin_lock(&send_cq->lock);
hns_roce_spin_lock(&recv_cq->hr_lock);
hns_roce_spin_lock(&send_cq->hr_lock);
}
} else if (send_cq) {
pthread_spin_lock(&send_cq->lock);
hns_roce_spin_lock(&send_cq->hr_lock);
} else if (recv_cq) {
pthread_spin_lock(&recv_cq->lock);
hns_roce_spin_lock(&recv_cq->hr_lock);
}
}

Expand All @@ -1633,18 +1633,18 @@ static void hns_roce_unlock_cqs(struct ibv_qp *qp)

if (send_cq && recv_cq) {
if (send_cq == recv_cq) {
pthread_spin_unlock(&send_cq->lock);
hns_roce_spin_unlock(&send_cq->hr_lock);
} else if (send_cq->cqn < recv_cq->cqn) {
pthread_spin_unlock(&recv_cq->lock);
pthread_spin_unlock(&send_cq->lock);
hns_roce_spin_unlock(&recv_cq->hr_lock);
hns_roce_spin_unlock(&send_cq->hr_lock);
} else {
pthread_spin_unlock(&send_cq->lock);
pthread_spin_unlock(&recv_cq->lock);
hns_roce_spin_unlock(&send_cq->hr_lock);
hns_roce_spin_unlock(&recv_cq->hr_lock);
}
} else if (send_cq) {
pthread_spin_unlock(&send_cq->lock);
hns_roce_spin_unlock(&send_cq->hr_lock);
} else if (recv_cq) {
pthread_spin_unlock(&recv_cq->lock);
hns_roce_spin_unlock(&recv_cq->hr_lock);
}
}

Expand Down Expand Up @@ -1816,11 +1816,11 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current,
if (attr->comp_mask)
return EINVAL;

pthread_spin_lock(&cq->lock);
hns_roce_spin_lock(&cq->hr_lock);

err = hns_roce_poll_one(ctx, &qp, cq, NULL);
if (err != V2_CQ_OK)
pthread_spin_unlock(&cq->lock);
hns_roce_spin_unlock(&cq->hr_lock);

return err;
}
Expand Down Expand Up @@ -1854,7 +1854,7 @@ static void wc_end_poll_cq(struct ibv_cq_ex *current)
else
update_cq_db(ctx, cq);

pthread_spin_unlock(&cq->lock);
hns_roce_spin_unlock(&cq->hr_lock);
}

static enum ibv_wc_opcode wc_read_opcode(struct ibv_cq_ex *current)
Expand Down
41 changes: 37 additions & 4 deletions providers/hns/hns_roce_u_verbs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,11 @@ int hns_roce_u_dealloc_mw(struct ibv_mw *mw)
return 0;
}

enum {
CREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS |
IBV_CQ_INIT_ATTR_MASK_PD,
};

enum {
CREATE_CQ_SUPPORTED_WC_FLAGS = IBV_WC_STANDARD_FLAGS |
IBV_WC_EX_WITH_CVLAN,
Expand All @@ -410,21 +415,42 @@ enum {
static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr,
struct hns_roce_context *context)
{
struct hns_roce_pad *pad = to_hr_pad(attr->parent_domain);

if (!attr->cqe || attr->cqe > context->max_cqe)
return EINVAL;

if (attr->comp_mask)
if (!check_comp_mask(attr->comp_mask, CREATE_CQ_SUPPORTED_COMP_MASK)) {
verbs_err(&context->ibv_ctx, "unsupported cq comps 0x%x\n",
attr->comp_mask);
return EOPNOTSUPP;
}

if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS))
return EOPNOTSUPP;

if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD) {
if (!pad) {
verbs_err(&context->ibv_ctx, "failed to check the pad of cq.\n");
return EINVAL;
}
atomic_fetch_add(&pad->pd.refcount, 1);
}

attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM,
roundup_pow_of_two(attr->cqe));

return 0;
}

static int hns_roce_cq_spinlock_init(struct hns_roce_cq *cq,
struct ibv_cq_init_attr_ex *attr)
{
bool need_lock = hns_roce_whether_need_lock(attr->parent_domain);

return hns_roce_spinlock_init(&cq->hr_lock, need_lock);
}

static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq)
{
int buf_size = hr_hw_page_align(cq->cq_depth * cq->cqe_size);
Expand Down Expand Up @@ -481,7 +507,10 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context,
goto err;
}

ret = pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
if (attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_PD)
cq->parent_domain = attr->parent_domain;

ret = hns_roce_cq_spinlock_init(cq, attr);
if (ret)
goto err_lock;

Expand Down Expand Up @@ -511,7 +540,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context,
err_db:
hns_roce_free_buf(&cq->buf);
err_buf:
pthread_spin_destroy(&cq->lock);
hns_roce_spinlock_destroy(&cq->hr_lock);
err_lock:
free(cq);
err:
Expand Down Expand Up @@ -564,6 +593,7 @@ int hns_roce_u_modify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr)
int hns_roce_u_destroy_cq(struct ibv_cq *cq)
{
struct hns_roce_cq *hr_cq = to_hr_cq(cq);
struct hns_roce_pad *pad = to_hr_pad(hr_cq->parent_domain);
int ret;

ret = ibv_cmd_destroy_cq(cq);
Expand All @@ -574,7 +604,10 @@ int hns_roce_u_destroy_cq(struct ibv_cq *cq)
HNS_ROCE_CQ_TYPE_DB);
hns_roce_free_buf(&hr_cq->buf);

pthread_spin_destroy(&hr_cq->lock);
hns_roce_spinlock_destroy(&hr_cq->hr_lock);

if (pad)
atomic_fetch_sub(&pad->pd.refcount, 1);

free(hr_cq);

Expand Down

0 comments on commit 8c865c3

Please sign in to comment.