Commit 30b87133 authored by Kalesh AP's avatar Kalesh AP Committed by Leon Romanovsky
Browse files

RDMA/bnxt_re: Refactor NQ allocation 



Move NQ related data structures from rdev to a new structure
named "struct bnxt_re_nq_record" by keeping a pointer to in
the rdev structure. Allocate the memory for it dynamically.
This change is needed for subsequent patches in the series.

Also, removed the nq_task variable from rdev structure as it
is redundant and no longer used.

This change would help to reduce the size of the driver private
structure as well.

Reviewed-by: default avatarChandramohan Akula <chandramohan.akula@broadcom.com>
Signed-off-by: default avatarKalesh AP <kalesh-anakkur.purayil@broadcom.com>
Signed-off-by: default avatarSelvin Xavier <selvin.xavier@broadcom.com>
Link: https://patch.msgid.link/1731577748-1804-3-git-send-email-selvin.xavier@broadcom.com


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 65ecee13

drivers/infiniband/hw/bnxt_re/bnxt_re.h

+7 −6
Original line number Diff line number Diff line
@@ -155,6 +155,11 @@ struct bnxt_re_pacing { 
#define BNXT_RE_GRC_FIFO_REG_BASE 0x2000



#define BNXT_RE_MIN_MSIX		2

#define BNXT_RE_MAX_MSIX		BNXT_MAX_ROCE_MSIX

struct bnxt_re_nq_record {

	struct bnxt_qplib_nq	nq[BNXT_RE_MAX_MSIX];

	int			num_msix;

};



#define MAX_CQ_HASH_BITS		(16)

#define MAX_SRQ_HASH_BITS		(16)
@@ -183,21 +188,17 @@ struct bnxt_re_dev { 
	unsigned int			version, major, minor;

	struct bnxt_qplib_chip_ctx	*chip_ctx;

	struct bnxt_en_dev		*en_dev;

	int				num_msix;



	int				id;



	struct delayed_work		worker;

	u8				cur_prio_map;



	/* FP Notification Queue (CQ & SRQ) */

	struct tasklet_struct		nq_task;



	/* RCFW Channel */

	struct bnxt_qplib_rcfw		rcfw;



	/* NQ */

	struct bnxt_qplib_nq		nq[BNXT_MAX_ROCE_MSIX];

	/* NQ record */

	struct bnxt_re_nq_record	*nqr;



	/* Device Resources */

	struct bnxt_qplib_dev_attr	dev_attr;

drivers/infiniband/hw/bnxt_re/ib_verbs.c

+3 −3
Original line number Diff line number Diff line
@@ -1872,8 +1872,8 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, 
	srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);

	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;

	srq->srq_limit = srq_init_attr->attr.srq_limit;

	srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;

	nq = &rdev->nq[0];

	srq->qplib_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;

	nq = &rdev->nqr->nq[0];



	if (udata) {

		rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
@@ -3122,7 +3122,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 
	 * used for getting the NQ index.

	 */

	nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);

	nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];

	nq = &rdev->nqr->nq[nq_alloc_cnt % (rdev->nqr->num_msix - 1)];

	cq->qplib_cq.max_wqe = entries;

	cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;

	cq->qplib_cq.nq	= nq;

drivers/infiniband/hw/bnxt_re/main.c

+50 −24
Original line number Diff line number Diff line
@@ -326,8 +326,8 @@ static void bnxt_re_stop_irq(void *handle) 
	rdev = en_info->rdev;

	rcfw = &rdev->rcfw;



	for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) {

		nq = &rdev->nq[indx - 1];

	for (indx = BNXT_RE_NQ_IDX; indx < rdev->nqr->num_msix; indx++) {

		nq = &rdev->nqr->nq[indx - 1];

		bnxt_qplib_nq_stop_irq(nq, false);

	}
@@ -362,7 +362,7 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) 
	/* Vectors may change after restart, so update with new vectors

	 * in device sctructure.

	 */

	for (indx = 0; indx < rdev->num_msix; indx++)

	for (indx = 0; indx < rdev->nqr->num_msix; indx++)

		rdev->en_dev->msix_entries[indx].vector = ent[indx].vector;



	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
@@ -371,8 +371,8 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) 
		ibdev_warn(&rdev->ibdev, "Failed to reinit CREQ\n");

		return;

	}

	for (indx = BNXT_RE_NQ_IDX ; indx < rdev->num_msix; indx++) {

		nq = &rdev->nq[indx - 1];

	for (indx = BNXT_RE_NQ_IDX ; indx < rdev->nqr->num_msix; indx++) {

		nq = &rdev->nqr->nq[indx - 1];

		rc = bnxt_qplib_nq_start_irq(nq, indx - 1,

					     msix_ent[indx].vector, false);

		if (rc) {
@@ -1206,7 +1206,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) 


	addrconf_addr_eui48((u8 *)&ibdev->node_guid, rdev->netdev->dev_addr);



	ibdev->num_comp_vectors	= rdev->num_msix - 1;

	ibdev->num_comp_vectors	= rdev->nqr->num_msix - 1;

	ibdev->dev.parent = &rdev->en_dev->pdev->dev;

	ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
@@ -1551,8 +1551,8 @@ static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) 
{

	int i;



	for (i = 1; i < rdev->num_msix; i++)

		bnxt_qplib_disable_nq(&rdev->nq[i - 1]);

	for (i = 1; i < rdev->nqr->num_msix; i++)

		bnxt_qplib_disable_nq(&rdev->nqr->nq[i - 1]);



	if (rdev->qplib_res.rcfw)

		bnxt_qplib_cleanup_res(&rdev->qplib_res);
@@ -1566,9 +1566,9 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev) 


	bnxt_qplib_init_res(&rdev->qplib_res);



	for (i = 1; i < rdev->num_msix ; i++) {

	for (i = 1; i < rdev->nqr->num_msix ; i++) {

		db_offt = rdev->en_dev->msix_entries[i].db_offset;

		rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],

		rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nqr->nq[i - 1],

					  i - 1, rdev->en_dev->msix_entries[i].vector,

					  db_offt, &bnxt_re_cqn_handler,

					  &bnxt_re_srqn_handler);
@@ -1582,20 +1582,22 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev) 
	return 0;

fail:

	for (i = num_vec_enabled; i >= 0; i--)

		bnxt_qplib_disable_nq(&rdev->nq[i]);

		bnxt_qplib_disable_nq(&rdev->nqr->nq[i]);

	return rc;

}



static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)

{

	struct bnxt_qplib_nq *nq;

	u8 type;

	int i;



	for (i = 0; i < rdev->num_msix - 1; i++) {

	for (i = 0; i < rdev->nqr->num_msix - 1; i++) {

		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);

		bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);

		bnxt_qplib_free_nq(&rdev->nq[i]);

		rdev->nq[i].res = NULL;

		nq = &rdev->nqr->nq[i];

		bnxt_re_net_ring_free(rdev, nq->ring_id, type);

		bnxt_qplib_free_nq(nq);

		nq->res = NULL;

	}

}
@@ -1637,12 +1639,12 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) 
	if (rc)

		goto dealloc_res;



	for (i = 0; i < rdev->num_msix - 1; i++) {

	for (i = 0; i < rdev->nqr->num_msix - 1; i++) {

		struct bnxt_qplib_nq *nq;



		nq = &rdev->nq[i];

		nq = &rdev->nqr->nq[i];

		nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;

		rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, &rdev->nq[i]);

		rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, nq);

		if (rc) {

			ibdev_err(&rdev->ibdev, "Alloc Failed NQ%d rc:%#x",

				  i, rc);
@@ -1650,7 +1652,7 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) 
		}

		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);

		rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;

		rattr.pages = nq->hwq.pbl[rdev->nq[i].hwq.level].pg_count;

		rattr.pages = nq->hwq.pbl[rdev->nqr->nq[i].hwq.level].pg_count;

		rattr.type = type;

		rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;

		rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
@@ -1660,7 +1662,7 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) 
			ibdev_err(&rdev->ibdev,

				  "Failed to allocate NQ fw id with rc = 0x%x",

				  rc);

			bnxt_qplib_free_nq(&rdev->nq[i]);

			bnxt_qplib_free_nq(nq);

			goto free_nq;

		}

		num_vec_created++;
@@ -1669,8 +1671,8 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) 
free_nq:

	for (i = num_vec_created - 1; i >= 0; i--) {

		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);

		bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);

		bnxt_qplib_free_nq(&rdev->nq[i]);

		bnxt_re_net_ring_free(rdev, rdev->nqr->nq[i].ring_id, type);

		bnxt_qplib_free_nq(&rdev->nqr->nq[i]);

	}

	bnxt_qplib_dealloc_dpi(&rdev->qplib_res,

			       &rdev->dpi_privileged);
@@ -1865,6 +1867,21 @@ static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) 
	return rc;

}



static int bnxt_re_alloc_nqr_mem(struct bnxt_re_dev *rdev)

{

	rdev->nqr = kzalloc(sizeof(*rdev->nqr), GFP_KERNEL);

	if (!rdev->nqr)

		return -ENOMEM;



	return 0;

}



static void bnxt_re_free_nqr_mem(struct bnxt_re_dev *rdev)

{

	kfree(rdev->nqr);

	rdev->nqr = NULL;

}



static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)

{

	u8 type;
@@ -1894,11 +1911,12 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) 
		bnxt_qplib_free_rcfw_channel(&rdev->rcfw);

	}



	rdev->num_msix = 0;

	rdev->nqr->num_msix = 0;



	if (rdev->pacing.dbr_pacing)

		bnxt_re_deinitialize_dbr_pacing(rdev);



	bnxt_re_free_nqr_mem(rdev);

	bnxt_re_destroy_chip_ctx(rdev);

	if (op_type == BNXT_RE_COMPLETE_REMOVE) {

		if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
@@ -1946,7 +1964,6 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type) 
	}

	ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",

		  rdev->en_dev->ulp_tbl->msix_requested);

	rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested;



	rc = bnxt_re_setup_chip_ctx(rdev);

	if (rc) {
@@ -1956,6 +1973,15 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type) 
		return -EINVAL;

	}



	rc = bnxt_re_alloc_nqr_mem(rdev);

	if (rc) {

		bnxt_re_destroy_chip_ctx(rdev);

		bnxt_unregister_dev(rdev->en_dev);

		clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);

		return rc;

	}

	rdev->nqr->num_msix = rdev->en_dev->ulp_tbl->msix_requested;



	/* Check whether VF or PF */

	bnxt_re_get_sriov_func_type(rdev);