Commit f3bdbd42 authored by Abhijit Gangurde's avatar Abhijit Gangurde Committed by Leon Romanovsky
Browse files

RDMA/ionic: Create device queues to support admin operations 



Setup RDMA admin queues using device command exposed over
auxiliary device and manage these queues using ida.

Co-developed-by: default avatarAndrew Boyer <andrew.boyer@amd.com>
Signed-off-by: default avatarAndrew Boyer <andrew.boyer@amd.com>
Co-developed-by: default avatarAllen Hubbe <allen.hubbe@amd.com>
Signed-off-by: default avatarAllen Hubbe <allen.hubbe@amd.com>
Signed-off-by: default avatarAbhijit Gangurde <abhijit.gangurde@amd.com>
Link: https://patch.msgid.link/20250903061606.4139957-10-abhijit.gangurde@amd.com


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 8d765af5

drivers/infiniband/hw/ionic/ionic_admin.c

0 → 100644
+1124 −0

File added.

Preview size limit exceeded, changes collapsed.

drivers/infiniband/hw/ionic/ionic_controlpath.c

0 → 100644
+181 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */



#include "ionic_ibdev.h"



static int ionic_validate_qdesc(struct ionic_qdesc *q)

{

	if (!q->addr || !q->size || !q->mask ||

	    !q->depth_log2 || !q->stride_log2)

		return -EINVAL;



	if (q->addr & (PAGE_SIZE - 1))

		return -EINVAL;



	if (q->mask != BIT(q->depth_log2) - 1)

		return -EINVAL;



	if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2))

		return -EINVAL;



	return 0;

}



static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx)

{

	/* EQ per vector per udma, and the first eqs reserved for async events.

	 * The rest of the vectors can be requested for completions.

	 */

	u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1;



	return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx;

}



static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx)

{

	unsigned int size, base, bound;

	int rc;



	size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count;

	base = size * udma_idx;

	bound = base + size;



	rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound);

	if (rc >= 0) {

		/* cq_base is zero or a multiple of two queue groups */

		*cqid = dev->lif_cfg.cq_base +

			ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift,

					   dev->half_cqid_udma_shift);



		rc = 0;

	}



	return rc;

}



static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid)

{

	u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base,

				       dev->lif_cfg.udma_qgrp_shift,

				       dev->half_cqid_udma_shift);



	ionic_resid_put(&dev->inuse_cqid, bitid);

}



int ionic_create_cq_common(struct ionic_vcq *vcq,

			   struct ionic_tbl_buf *buf,

			   const struct ib_cq_init_attr *attr,

			   struct ionic_ctx *ctx,

			   struct ib_udata *udata,

			   struct ionic_qdesc *req_cq,

			   __u32 *resp_cqid,

			   int udma_idx)

{

	struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);

	struct ionic_cq *cq = &vcq->cq[udma_idx];

	void *entry;

	int rc;



	cq->vcq = vcq;



	if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) {

		rc = -EINVAL;

		goto err_args;

	}



	rc = ionic_get_cqid(dev, &cq->cqid, udma_idx);

	if (rc)

		goto err_args;



	cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx);



	spin_lock_init(&cq->lock);

	INIT_LIST_HEAD(&cq->poll_sq);

	INIT_LIST_HEAD(&cq->flush_sq);

	INIT_LIST_HEAD(&cq->flush_rq);



	if (udata) {

		rc = ionic_validate_qdesc(req_cq);

		if (rc)

			goto err_qdesc;



		cq->umem = ib_umem_get(&dev->ibdev, req_cq->addr, req_cq->size,

				       IB_ACCESS_LOCAL_WRITE);

		if (IS_ERR(cq->umem)) {

			rc = PTR_ERR(cq->umem);

			goto err_qdesc;

		}



		cq->q.ptr = NULL;

		cq->q.size = req_cq->size;

		cq->q.mask = req_cq->mask;

		cq->q.depth_log2 = req_cq->depth_log2;

		cq->q.stride_log2 = req_cq->stride_log2;



		*resp_cqid = cq->cqid;

	} else {

		rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev,

				      attr->cqe + IONIC_CQ_GRACE,

				      sizeof(struct ionic_v1_cqe));

		if (rc)

			goto err_q_init;



		ionic_queue_dbell_init(&cq->q, cq->cqid);

		cq->color = true;

		cq->credit = cq->q.mask;

	}



	rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE);

	if (rc)

		goto err_pgtbl_init;



	init_completion(&cq->cq_rel_comp);

	kref_init(&cq->cq_kref);



	entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);

	if (entry) {

		if (!xa_is_err(entry))

			rc = -EINVAL;

		else

			rc = xa_err(entry);



		goto err_xa;

	}



	return 0;



err_xa:

	ionic_pgtbl_unbuf(dev, buf);

err_pgtbl_init:

	if (!udata)

		ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);

err_q_init:

	if (cq->umem)

		ib_umem_release(cq->umem);

err_qdesc:

	ionic_put_cqid(dev, cq->cqid);

err_args:

	cq->vcq = NULL;



	return rc;

}



void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq)

{

	if (!cq->vcq)

		return;



	xa_erase_irq(&dev->cq_tbl, cq->cqid);



	kref_put(&cq->cq_kref, ionic_cq_complete);

	wait_for_completion(&cq->cq_rel_comp);



	if (cq->umem)

		ib_umem_release(cq->umem);

	else

		ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);



	ionic_put_cqid(dev, cq->cqid);



	cq->vcq = NULL;

}

drivers/infiniband/hw/ionic/ionic_fw.h

0 → 100644
+164 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */



#ifndef _IONIC_FW_H_

#define _IONIC_FW_H_



#include <linux/kernel.h>



/* completion queue v1 cqe */

struct ionic_v1_cqe {

	union {

		struct {

			__be16		cmd_idx;

			__u8		cmd_op;

			__u8		rsvd[17];

			__le16		old_sq_cindex;

			__le16		old_rq_cq_cindex;

		} admin;

		struct {

			__u64		wqe_id;

			__be32		src_qpn_op;

			__u8		src_mac[6];

			__be16		vlan_tag;

			__be32		imm_data_rkey;

		} recv;

		struct {

			__u8		rsvd[4];

			__be32		msg_msn;

			__u8		rsvd2[8];

			__u64		npg_wqe_id;

		} send;

	};

	__be32				status_length;

	__be32				qid_type_flags;

};



/* bits for cqe qid_type_flags */

enum ionic_v1_cqe_qtf_bits {

	IONIC_V1_CQE_COLOR		= BIT(0),

	IONIC_V1_CQE_ERROR		= BIT(1),

	IONIC_V1_CQE_TYPE_SHIFT		= 5,

	IONIC_V1_CQE_TYPE_MASK		= 0x7,

	IONIC_V1_CQE_QID_SHIFT		= 8,



	IONIC_V1_CQE_TYPE_ADMIN		= 0,

	IONIC_V1_CQE_TYPE_RECV		= 1,

	IONIC_V1_CQE_TYPE_SEND_MSN	= 2,

	IONIC_V1_CQE_TYPE_SEND_NPG	= 3,

};



static inline bool ionic_v1_cqe_color(struct ionic_v1_cqe *cqe)

{

	return cqe->qid_type_flags & cpu_to_be32(IONIC_V1_CQE_COLOR);

}



static inline bool ionic_v1_cqe_error(struct ionic_v1_cqe *cqe)

{

	return cqe->qid_type_flags & cpu_to_be32(IONIC_V1_CQE_ERROR);

}



static inline void ionic_v1_cqe_clean(struct ionic_v1_cqe *cqe)

{

	cqe->qid_type_flags |= cpu_to_be32(~0u << IONIC_V1_CQE_QID_SHIFT);

}



static inline u32 ionic_v1_cqe_qtf(struct ionic_v1_cqe *cqe)

{

	return be32_to_cpu(cqe->qid_type_flags);

}



static inline u8 ionic_v1_cqe_qtf_type(u32 qtf)

{

	return (qtf >> IONIC_V1_CQE_TYPE_SHIFT) & IONIC_V1_CQE_TYPE_MASK;

}



static inline u32 ionic_v1_cqe_qtf_qid(u32 qtf)

{

	return qtf >> IONIC_V1_CQE_QID_SHIFT;

}



#define ADMIN_WQE_STRIDE	64

#define ADMIN_WQE_HDR_LEN	4



/* admin queue v1 wqe */

struct ionic_v1_admin_wqe {

	__u8				op;

	__u8				rsvd;

	__le16				len;



	union {

	} cmd;

};



/* admin queue v1 cqe status */

enum ionic_v1_admin_status {

	IONIC_V1_ASTS_OK,

	IONIC_V1_ASTS_BAD_CMD,

	IONIC_V1_ASTS_BAD_INDEX,

	IONIC_V1_ASTS_BAD_STATE,

	IONIC_V1_ASTS_BAD_TYPE,

	IONIC_V1_ASTS_BAD_ATTR,

	IONIC_V1_ASTS_MSG_TOO_BIG,

};



/* event queue v1 eqe */

struct ionic_v1_eqe {

	__be32				evt;

};



/* bits for cqe queue_type_flags */

enum ionic_v1_eqe_evt_bits {

	IONIC_V1_EQE_COLOR		= BIT(0),

	IONIC_V1_EQE_TYPE_SHIFT		= 1,

	IONIC_V1_EQE_TYPE_MASK		= 0x7,

	IONIC_V1_EQE_CODE_SHIFT		= 4,

	IONIC_V1_EQE_CODE_MASK		= 0xf,

	IONIC_V1_EQE_QID_SHIFT		= 8,



	/* cq events */

	IONIC_V1_EQE_TYPE_CQ		= 0,

	/* cq normal events */

	IONIC_V1_EQE_CQ_NOTIFY		= 0,

	/* cq error events */

	IONIC_V1_EQE_CQ_ERR		= 8,



	/* qp and srq events */

	IONIC_V1_EQE_TYPE_QP		= 1,

	/* qp normal events */

	IONIC_V1_EQE_SRQ_LEVEL		= 0,

	IONIC_V1_EQE_SQ_DRAIN		= 1,

	IONIC_V1_EQE_QP_COMM_EST	= 2,

	IONIC_V1_EQE_QP_LAST_WQE	= 3,

	/* qp error events */

	IONIC_V1_EQE_QP_ERR		= 8,

	IONIC_V1_EQE_QP_ERR_REQUEST	= 9,

	IONIC_V1_EQE_QP_ERR_ACCESS	= 10,

};



static inline bool ionic_v1_eqe_color(struct ionic_v1_eqe *eqe)

{

	return eqe->evt & cpu_to_be32(IONIC_V1_EQE_COLOR);

}



static inline u32 ionic_v1_eqe_evt(struct ionic_v1_eqe *eqe)

{

	return be32_to_cpu(eqe->evt);

}



static inline u8 ionic_v1_eqe_evt_type(u32 evt)

{

	return (evt >> IONIC_V1_EQE_TYPE_SHIFT) & IONIC_V1_EQE_TYPE_MASK;

}



static inline u8 ionic_v1_eqe_evt_code(u32 evt)

{

	return (evt >> IONIC_V1_EQE_CODE_SHIFT) & IONIC_V1_EQE_CODE_MASK;

}



static inline u32 ionic_v1_eqe_evt_qid(u32 evt)

{

	return evt >> IONIC_V1_EQE_QID_SHIFT;

}



#endif /* _IONIC_FW_H_ */

drivers/infiniband/hw/ionic/ionic_ibdev.c

+56 −0
Original line number Diff line number Diff line
@@ -15,9 +15,41 @@ MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 
MODULE_LICENSE("GPL");

MODULE_IMPORT_NS("NET_IONIC");



static void ionic_init_resids(struct ionic_ibdev *dev)

{

	ionic_resid_init(&dev->inuse_cqid, dev->lif_cfg.cq_count);

	dev->half_cqid_udma_shift =

		order_base_2(dev->lif_cfg.cq_count / dev->lif_cfg.udma_count);

	ionic_resid_init(&dev->inuse_pdid, IONIC_MAX_PD);

	ionic_resid_init(&dev->inuse_ahid, dev->lif_cfg.nahs_per_lif);

	ionic_resid_init(&dev->inuse_mrid, dev->lif_cfg.nmrs_per_lif);

	/* skip reserved lkey */

	dev->next_mrkey = 1;

	ionic_resid_init(&dev->inuse_qpid, dev->lif_cfg.qp_count);

	/* skip reserved SMI and GSI qpids */

	dev->half_qpid_udma_shift =

		order_base_2(dev->lif_cfg.qp_count / dev->lif_cfg.udma_count);

	ionic_resid_init(&dev->inuse_dbid, dev->lif_cfg.dbid_count);

}



static void ionic_destroy_resids(struct ionic_ibdev *dev)

{

	ionic_resid_destroy(&dev->inuse_cqid);

	ionic_resid_destroy(&dev->inuse_pdid);

	ionic_resid_destroy(&dev->inuse_ahid);

	ionic_resid_destroy(&dev->inuse_mrid);

	ionic_resid_destroy(&dev->inuse_qpid);

	ionic_resid_destroy(&dev->inuse_dbid);

}



static void ionic_destroy_ibdev(struct ionic_ibdev *dev)

{

	ionic_kill_rdma_admin(dev, false);

	ib_unregister_device(&dev->ibdev);

	ionic_destroy_rdma_admin(dev);

	ionic_destroy_resids(dev);

	WARN_ON(!xa_empty(&dev->cq_tbl));

	xa_destroy(&dev->cq_tbl);

	ib_dealloc_device(&dev->ibdev);

}
@@ -34,6 +66,18 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) 


	ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg);



	xa_init_flags(&dev->cq_tbl, GFP_ATOMIC);



	ionic_init_resids(dev);



	rc = ionic_rdma_reset_devcmd(dev);

	if (rc)

		goto err_reset;



	rc = ionic_create_rdma_admin(dev);

	if (rc)

		goto err_admin;



	ibdev = &dev->ibdev;

	ibdev->dev.parent = dev->lif_cfg.hwdev;
@@ -62,6 +106,11 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) 


err_register:

err_admin:

	ionic_kill_rdma_admin(dev, false);

	ionic_destroy_rdma_admin(dev);

err_reset:

	ionic_destroy_resids(dev);

	xa_destroy(&dev->cq_tbl);

	ib_dealloc_device(&dev->ibdev);



	return ERR_PTR(rc);
@@ -112,6 +161,10 @@ static int __init ionic_mod_init(void) 
{

	int rc;



	ionic_evt_workq = create_workqueue(DRIVER_NAME "-evt");

	if (!ionic_evt_workq)

		return -ENOMEM;



	rc = auxiliary_driver_register(&ionic_aux_r_driver);

	if (rc)

		goto err_aux;
@@ -119,12 +172,15 @@ static int __init ionic_mod_init(void) 
	return 0;



err_aux:

	destroy_workqueue(ionic_evt_workq);



	return rc;

}



static void __exit ionic_mod_exit(void)

{

	auxiliary_driver_unregister(&ionic_aux_r_driver);

	destroy_workqueue(ionic_evt_workq);

}



module_init(ionic_mod_init);

drivers/infiniband/hw/ionic/ionic_ibdev.h

+222 −0
Original line number Diff line number Diff line
@@ -4,15 +4,237 @@ 
#ifndef _IONIC_IBDEV_H_

#define _IONIC_IBDEV_H_



#include <rdma/ib_umem.h>

#include <rdma/ib_verbs.h>



#include <ionic_api.h>

#include <ionic_regs.h>



#include "ionic_fw.h"

#include "ionic_queue.h"

#include "ionic_res.h"



#include "ionic_lif_cfg.h"



/* Config knobs */

#define IONIC_EQ_DEPTH 511

#define IONIC_EQ_COUNT 32

#define IONIC_AQ_DEPTH 63

#define IONIC_AQ_COUNT 4

#define IONIC_EQ_ISR_BUDGET 10

#define IONIC_EQ_WORK_BUDGET 1000

#define IONIC_MAX_PD 1024



#define IONIC_CQ_GRACE 100



struct ionic_aq;

struct ionic_cq;

struct ionic_eq;

struct ionic_vcq;



enum ionic_admin_state {

	IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */

	IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */

	IONIC_ADMIN_KILLED, /* not submitting, locally completed */

};



enum ionic_admin_flags {

	IONIC_ADMIN_F_BUSYWAIT  = BIT(0),	/* Don't sleep */

	IONIC_ADMIN_F_TEARDOWN  = BIT(1),	/* In destroy path */

	IONIC_ADMIN_F_INTERRUPT = BIT(2),	/* Interruptible w/timeout */

};



struct ionic_qdesc {

	__aligned_u64 addr;

	__u32 size;

	__u16 mask;

	__u8 depth_log2;

	__u8 stride_log2;

};



enum ionic_mmap_flag {

	IONIC_MMAP_WC = BIT(0),

};



struct ionic_mmap_entry {

	struct rdma_user_mmap_entry rdma_entry;

	unsigned long size;

	unsigned long pfn;

	u8 mmap_flags;

};



struct ionic_ibdev {

	struct ib_device	ibdev;



	struct ionic_lif_cfg	lif_cfg;



	struct xarray		qp_tbl;

	struct xarray		cq_tbl;



	struct ionic_resid_bits	inuse_dbid;

	struct ionic_resid_bits	inuse_pdid;

	struct ionic_resid_bits	inuse_ahid;

	struct ionic_resid_bits	inuse_mrid;

	struct ionic_resid_bits	inuse_qpid;

	struct ionic_resid_bits	inuse_cqid;



	u8			half_cqid_udma_shift;

	u8			half_qpid_udma_shift;

	u8			next_qpid_udma_idx;

	u8			next_mrkey;



	struct work_struct	reset_work;

	bool			reset_posted;

	u32			reset_cnt;



	struct delayed_work	admin_dwork;

	struct ionic_aq		**aq_vec;

	atomic_t		admin_state;



	struct ionic_eq		**eq_vec;

};



struct ionic_eq {

	struct ionic_ibdev	*dev;



	u32			eqid;

	u32			intr;



	struct ionic_queue	q;



	bool			armed;

	bool			enable;



	struct work_struct	work;



	int			irq;

	char			name[32];

};



struct ionic_admin_wr {

	struct completion		work;

	struct list_head		aq_ent;

	struct ionic_v1_admin_wqe	wqe;

	struct ionic_v1_cqe		cqe;

	struct ionic_aq			*aq;

	int				status;

};



struct ionic_admin_wr_q {

	struct ionic_admin_wr	*wr;

	int			wqe_strides;

};



struct ionic_aq {

	struct ionic_ibdev	*dev;

	struct ionic_vcq	*vcq;



	struct work_struct	work;



	atomic_t		admin_state;

	unsigned long		stamp;

	bool			armed;



	u32			aqid;

	u32			cqid;



	spinlock_t		lock; /* for posting */

	struct ionic_queue	q;

	struct ionic_admin_wr_q	*q_wr;

	struct list_head	wr_prod;

	struct list_head	wr_post;

};



struct ionic_ctx {

	struct ib_ucontext	ibctx;

	u32			dbid;

	struct rdma_user_mmap_entry	*mmap_dbell;

};



struct ionic_tbl_buf {

	u32		tbl_limit;

	u32		tbl_pages;

	size_t		tbl_size;

	__le64		*tbl_buf;

	dma_addr_t	tbl_dma;

	u8		page_size_log2;

};



struct ionic_cq {

	struct ionic_vcq	*vcq;



	u32			cqid;

	u32			eqid;



	spinlock_t		lock; /* for polling */

	struct list_head	poll_sq;

	bool			flush;

	struct list_head	flush_sq;

	struct list_head	flush_rq;

	struct list_head	ibkill_flush_ent;



	struct ionic_queue	q;

	bool			color;

	int			credit;

	u16			arm_any_prod;

	u16			arm_sol_prod;



	struct kref		cq_kref;

	struct completion	cq_rel_comp;



	/* infrequently accessed, keep at end */

	struct ib_umem		*umem;

};



struct ionic_vcq {

	struct ib_cq		ibcq;

	struct ionic_cq		cq[2];

	u8			udma_mask;

	u8			poll_idx;

};



static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)

{

	return container_of(ibdev, struct ionic_ibdev, ibdev);

}



static inline void ionic_cq_complete(struct kref *kref)

{

	struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);



	complete(&cq->cq_rel_comp);

}



/* ionic_admin.c */

extern struct workqueue_struct *ionic_evt_workq;

void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);

int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,

		     enum ionic_admin_flags);



int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);



int ionic_create_rdma_admin(struct ionic_ibdev *dev);

void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);

void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);



/* ionic_controlpath.c */

int ionic_create_cq_common(struct ionic_vcq *vcq,

			   struct ionic_tbl_buf *buf,

			   const struct ib_cq_init_attr *attr,

			   struct ionic_ctx *ctx,

			   struct ib_udata *udata,

			   struct ionic_qdesc *req_cq,

			   __u32 *resp_cqid,

			   int udma_idx);

void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);



/* ionic_pgtbl.c */

int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);

int ionic_pgtbl_init(struct ionic_ibdev *dev,

		     struct ionic_tbl_buf *buf,

		     struct ib_umem *umem,

		     dma_addr_t dma,

		     int limit,

		     u64 page_size);

void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);

#endif /* _IONIC_IBDEV_H_ */