Commit aef3cdb4 authored by Halil Pasic's avatar Halil Pasic Committed by Paolo Abeni
Browse files

net/smc: make wr buffer count configurable 



Think SMC_WR_BUF_CNT_SEND := SMC_WR_BUF_CNT used in send context and
SMC_WR_BUF_CNT_RECV := 3 * SMC_WR_BUF_CNT used in recv context. Those
get replaced with lgr->max_send_wr and lgr->max_recv_wr respective.

Please note that although with the default sysctl values
qp_attr.cap.max_send_wr ==  qp_attr.cap.max_recv_wr is maintained but
can not be assumed to be generally true any more. I see no downside to
that, but my confidence level is rather modest.

Signed-off-by: default avatarHalil Pasic <pasic@linux.ibm.com>
Reviewed-by: default avatarSidraya Jayagond <sidraya@linux.ibm.com>
Reviewed-by: default avatarDust Li <dust.li@linux.alibaba.com>
Tested-by: default avatarMahanta Jambigi <mjambigi@linux.ibm.com>
Link: https://patch.msgid.link/20251027224856.2970019-2-pasic@linux.ibm.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent ea7d0d60

Documentation/networking/smc-sysctl.rst

+36 −0
Original line number Diff line number Diff line
@@ -71,3 +71,39 @@ smcr_max_conns_per_lgr - INTEGER 
	acceptable value ranges from 16 to 255. Only for SMC-R v2.1 and later.



	Default: 255



smcr_max_send_wr - INTEGER

	So-called work request buffers are SMCR link (and RDMA queue pair) level

	resources necessary for performing RDMA operations. Since up to 255

	connections can share a link group and thus also a link and the number

	of the work request buffers is decided when the link is allocated,

	depending on the workload it can be a bottleneck in a sense that threads

	have to wait for work request buffers to become available. Before the

	introduction of this control the maximal number of work request buffers

	available on the send path used to be hard coded to 16. With this control

	it becomes configurable. The acceptable range is between 2 and 2048.



	Please be aware that all the buffers need to be allocated as a physically

	continuous array in which each element is a single buffer and has the size

	of SMC_WR_BUF_SIZE (48) bytes. If the allocation fails we give up much

	like before having this control.



	Default: 16



smcr_max_recv_wr - INTEGER

	So-called work request buffers are SMCR link (and RDMA queue pair) level

	resources necessary for performing RDMA operations. Since up to 255

	connections can share a link group and thus also a link and the number

	of the work request buffers is decided when the link is allocated,

	depending on the workload it can be a bottleneck in a sense that threads

	have to wait for work request buffers to become available. Before the

	introduction of this control the maximal number of work request buffers

	available on the receive path used to be hard coded to 16. With this control

	it becomes configurable. The acceptable range is between 2 and 2048.



	Please be aware that all the buffers need to be allocated as a physically

	continuous array in which each element is a single buffer and has the size

	of SMC_WR_BUF_SIZE (48) bytes. If the allocation fails we give up much

	like before having this control.



	Default: 48

include/net/netns/smc.h

+2 −0
Original line number Diff line number Diff line
@@ -24,5 +24,7 @@ struct netns_smc { 
	int				sysctl_rmem;

	int				sysctl_max_links_per_lgr;

	int				sysctl_max_conns_per_lgr;

	unsigned int			sysctl_smcr_max_send_wr;

	unsigned int			sysctl_smcr_max_recv_wr;

};

#endif

net/smc/smc_core.h

+6 −0
Original line number Diff line number Diff line
@@ -34,6 +34,8 @@ 
					 * distributions may modify it to a value between

					 * 16-255 as needed.

					 */

#define SMCR_MAX_SEND_WR_DEF	16	/* Default number of work requests per send queue */

#define SMCR_MAX_RECV_WR_DEF	48	/* Default number of work requests per recv queue */



struct smc_lgr_list {			/* list of link group definition */

	struct list_head	list;
@@ -366,6 +368,10 @@ struct smc_link_group { 
						/* max conn can be assigned to lgr */

			u8			max_links;

						/* max links can be added in lgr */

			u16			max_send_wr;

						/* number of WR buffers on send */

			u16			max_recv_wr;

						/* number of WR buffers on recv */

		};

		struct { /* SMC-D */

			struct smcd_gid		peer_gid;

net/smc/smc_ib.c

+5 −5
Original line number Diff line number Diff line
@@ -669,11 +669,6 @@ int smc_ib_create_queue_pair(struct smc_link *lnk) 
		.recv_cq = lnk->smcibdev->roce_cq_recv,

		.srq = NULL,

		.cap = {

				/* include unsolicited rdma_writes as well,

				 * there are max. 2 RDMA_WRITE per 1 WR_SEND

				 */

			.max_send_wr = SMC_WR_BUF_CNT * 3,

			.max_recv_wr = SMC_WR_BUF_CNT * 3,

			.max_send_sge = SMC_IB_MAX_SEND_SGE,

			.max_recv_sge = lnk->wr_rx_sge_cnt,

			.max_inline_data = 0,
@@ -683,6 +678,11 @@ int smc_ib_create_queue_pair(struct smc_link *lnk) 
	};

	int rc;



	/* include unsolicited rdma_writes as well,

	 * there are max. 2 RDMA_WRITE per 1 WR_SEND

	 */

	qp_attr.cap.max_send_wr = 3 * lnk->lgr->max_send_wr;

	qp_attr.cap.max_recv_wr = lnk->lgr->max_recv_wr;

	lnk->roce_qp = ib_create_qp(lnk->roce_pd, &qp_attr);

	rc = PTR_ERR_OR_ZERO(lnk->roce_qp);

	if (IS_ERR(lnk->roce_qp))

net/smc/smc_llc.c

+2 −0
Original line number Diff line number Diff line
@@ -2157,6 +2157,8 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 
	init_waitqueue_head(&lgr->llc_msg_waiter);

	init_rwsem(&lgr->llc_conf_mutex);

	lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);

	lgr->max_send_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_send_wr));

	lgr->max_recv_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_recv_wr));

}



/* called after lgr was removed from lgr_list */