Commit e18fa0bb authored by Chiara Meiohas's avatar Chiara Meiohas Committed by Leon Romanovsky
Browse files

RDMA/core: Add an option to display driver-specific QPs in the rdmatool 



Utilize the -dd flag (driver-specific details) in the rdmatool
to view driver-specific QPs which are not exposed yet.

Add the netlink attribute to mark request to convey driver details and
use it to return QP subtype as a string.

$ rdma resource show qp link ibp8s0f1
link ibp8s0f1/1 lqpn 360 type UD state RTS sq-psn 0 comm [mlx5_ib]
link ibp8s0f1/1 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core]
link ibp8s0f1/1 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core]

$ rdma resource show qp link ibp8s0f1 -dd
link ibp8s0f1/1 lqpn 360 type UD state RTS sq-psn 0 comm [mlx5_ib]
link ibp8s0f1/1 lqpn 465 type DRIVER subtype REG_UMR state RTS sq-psn 0 comm [mlx5_ib]
link ibp8s0f1/1 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core]
link ibp8s0f1/1 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core]

$ rdma resource show
0: ibp8s0f0: pd 3 cq 4 qp 3 cm_id 0 mr 0 ctx 0 srq 2
1: ibp8s0f1: pd 3 cq 4 qp 3 cm_id 0 mr 0 ctx 0 srq 2

$ rdma resource show -dd
0: ibp8s0f0: pd 3 cq 4 qp 4 cm_id 0 mr 0 ctx 0 srq 2
1: ibp8s0f1: pd 3 cq 4 qp 4 cm_id 0 mr 0 ctx 0 srq 2

Signed-off-by: default avatarChiara Meiohas <cmeiohas@nvidia.com>
Link: https://lore.kernel.org/r/2607bb3ddec3cae3443c2ea19e9f700825d20a98.1713268997.git.leon@kernel.org


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent f847e840

drivers/infiniband/core/nldev.c

+19 −4
Original line number Diff line number Diff line
@@ -137,6 +137,8 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { 
	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING,

					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },

	[RDMA_NLDEV_ATTR_RES_TYPE]		= { .type = NLA_U8 },

	[RDMA_NLDEV_ATTR_RES_SUBTYPE]		= { .type = NLA_NUL_STRING,

					.len = RDMA_NLDEV_ATTR_EMPTY_STRING },

	[RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 },

	[RDMA_NLDEV_ATTR_RES_USECNT]		= { .type = NLA_U64 },

	[RDMA_NLDEV_ATTR_RES_SRQ]		= { .type = NLA_NESTED },
@@ -164,6 +166,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { 
	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX]	= { .type = NLA_U32 },

	[RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 },

	[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE] = { .type = NLA_U8 },

	[RDMA_NLDEV_ATTR_DRIVER_DETAILS]	= { .type = NLA_U8 },

};



static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -399,7 +402,8 @@ static int fill_res_info_entry(struct sk_buff *msg, 
	return -EMSGSIZE;

}



static int fill_res_info(struct sk_buff *msg, struct ib_device *device)

static int fill_res_info(struct sk_buff *msg, struct ib_device *device,

			 bool show_details)

{

	static const char * const names[RDMA_RESTRACK_MAX] = {

		[RDMA_RESTRACK_PD] = "pd",
@@ -424,7 +428,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device) 
	for (i = 0; i < RDMA_RESTRACK_MAX; i++) {

		if (!names[i])

			continue;

		curr = rdma_restrack_count(device, i);

		curr = rdma_restrack_count(device, i, show_details);

		ret = fill_res_info_entry(msg, names[i], curr);

		if (ret)

			goto err;
@@ -1305,6 +1309,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 
			      struct netlink_ext_ack *extack)

{

	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];

	bool show_details = false;

	struct ib_device *device;

	struct sk_buff *msg;

	u32 index;
@@ -1320,6 +1325,9 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 
	if (!device)

		return -EINVAL;



	if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS])

		show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]);



	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);

	if (!msg) {

		ret = -ENOMEM;
@@ -1334,7 +1342,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 
		goto err_free;

	}



	ret = fill_res_info(msg, device);

	ret = fill_res_info(msg, device, show_details);

	if (ret)

		goto err_free;
@@ -1364,7 +1372,7 @@ static int _nldev_res_get_dumpit(struct ib_device *device, 
			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),

			0, NLM_F_MULTI);



	if (!nlh || fill_res_info(skb, device)) {

	if (!nlh || fill_res_info(skb, device, false)) {

		nlmsg_cancel(skb, nlh);

		goto out;

	}
@@ -1534,6 +1542,7 @@ static int res_get_common_dumpit(struct sk_buff *skb, 
	struct rdma_restrack_entry *res;

	struct rdma_restrack_root *rt;

	int err, ret = 0, idx = 0;

	bool show_details = false;

	struct nlattr *table_attr;

	struct nlattr *entry_attr;

	struct ib_device *device;
@@ -1562,6 +1571,9 @@ static int res_get_common_dumpit(struct sk_buff *skb, 
	if (!device)

		return -EINVAL;



	if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS])

		show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]);



	/*

	 * If no PORT_INDEX is supplied, we will return all QPs from that device

	 */
@@ -1599,6 +1611,9 @@ static int res_get_common_dumpit(struct sk_buff *skb, 
	 * objects.

	 */

	xa_for_each(&rt->xa, id, res) {

		if (xa_get_mark(&rt->xa, res->id, RESTRACK_DD) && !show_details)

			goto next;



		if (idx < start || !rdma_restrack_get(res))

			goto next;

drivers/infiniband/core/restrack.c

+10 −2
Original line number Diff line number Diff line
@@ -59,8 +59,10 @@ void rdma_restrack_clean(struct ib_device *dev) 
 * rdma_restrack_count() - the current usage of specific object

 * @dev:  IB device

 * @type: actual type of object to operate

 * @show_details: count driver specific objects

 */

int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type)

int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,

			bool show_details)

{

	struct rdma_restrack_root *rt = &dev->res[type];

	struct rdma_restrack_entry *e;
@@ -68,8 +70,11 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) 
	u32 cnt = 0;



	xa_lock(&rt->xa);

	xas_for_each(&xas, e, U32_MAX)

	xas_for_each(&xas, e, U32_MAX) {

		if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details)

			continue;

		cnt++;

	}

	xa_unlock(&rt->xa);

	return cnt;

}
@@ -198,6 +203,9 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) 
		ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);

		if (ret)

			res->id = 0;



		if (qp->qp_type >= IB_QPT_DRIVER)

			xa_set_mark(&rt->xa, res->id, RESTRACK_DD);

	} else if (res->type == RDMA_RESTRACK_COUNTER) {

		/* Special case to ensure that cntn points to right counter */

		struct rdma_counter *counter;

include/rdma/restrack.h

+5 −2
Original line number Diff line number Diff line
@@ -14,6 +14,9 @@ 
#include <uapi/rdma/rdma_netlink.h>

#include <linux/xarray.h>



/* Mark entry as containing driver specific details, it is used to provide QP subtype for now */

#define RESTRACK_DD XA_MARK_1



struct ib_device;

struct sk_buff;
@@ -116,8 +119,8 @@ struct rdma_restrack_entry { 
	u32 id;

};



int rdma_restrack_count(struct ib_device *dev,

			enum rdma_restrack_type type);

int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,

			bool show_details);

/**

 * rdma_is_kernel_res() - check the owner of resource

 * @res:  resource entry

include/uapi/rdma/rdma_netlink.h

+6 −0
Original line number Diff line number Diff line
@@ -558,6 +558,12 @@ enum rdma_nldev_attr { 


	RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE, /* u8 */



	RDMA_NLDEV_ATTR_DRIVER_DETAILS,		/* u8 */

	/*

	 * QP subtype string, used for driver QPs

	 */

	RDMA_NLDEV_ATTR_RES_SUBTYPE,		/* string */



	/*

	 * Always the end

	 */