Commit a95fbe2a authored by Jack Wang's avatar Jack Wang Committed by Jason Gunthorpe
Browse files

RDMA/rtrs: Check device max_qp_wr limit when create QP 

Currently we only check device max_qp_wr limit for IO connection, but not
for service connection. We should check for both.

So save the max_qp_wr device limit in wr_limit, and use it for both IO
connections and service connections.

While at it, also remove an outdated comments.

Link: https://lore.kernel.org/r/20210614090337.29557-6-jinpu.wang@ionos.com


Suggested-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Signed-off-by: default avatarJack Wang <jinpu.wang@ionos.com>
Signed-off-by: default avatarGioh Kim <gi-oh.kim@ionos.com>
Reviewed-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 354462eb

drivers/infiniband/ulp/rtrs/rtrs-clt.c

+15 −14
Original line number Diff line number Diff line
@@ -1572,21 +1572,12 @@ static void destroy_con(struct rtrs_clt_con *con) 
static int create_con_cq_qp(struct rtrs_clt_con *con)

{

	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);

	u32 max_send_wr, max_recv_wr, cq_num, max_send_sge;

	u32 max_send_wr, max_recv_wr, cq_num, max_send_sge, wr_limit;

	int err, cq_vector;

	struct rtrs_msg_rkey_rsp *rsp;



	lockdep_assert_held(&con->con_mutex);

	if (con->c.cid == 0) {

		/*

		 * Two (request + registration) completion for send

		 * Two for recv if always_invalidate is set on server

		 * or one for recv.

		 * + 2 for drain and heartbeat

		 * in case qp gets into error state.

		 */

		max_send_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;

		max_recv_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;

		max_send_sge = 1;

		/* We must be the first here */

		if (WARN_ON(sess->s.dev))
@@ -1606,6 +1597,17 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) 
		}

		sess->s.dev_ref = 1;

		query_fast_reg_mode(sess);

		wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr;

		/*

		 * Two (request + registration) completion for send

		 * Two for recv if always_invalidate is set on server

		 * or one for recv.

		 * + 2 for drain and heartbeat

		 * in case qp gets into error state.

		 */

		max_send_wr =

			min_t(int, wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2);

		max_recv_wr = max_send_wr;

	} else {

		/*

		 * Here we assume that session members are correctly set.
@@ -1617,14 +1619,13 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) 
		if (WARN_ON(!sess->queue_depth))

			return -EINVAL;



		wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr;

		/* Shared between connections */

		sess->s.dev_ref++;

		max_send_wr =

			min_t(int, sess->s.dev->ib_dev->attrs.max_qp_wr,

		max_send_wr = min_t(int, wr_limit,

			      /* QD * (REQ + RSP + FR REGS or INVS) + drain */

			      sess->queue_depth * 3 + 1);

		max_recv_wr =

			min_t(int, sess->s.dev->ib_dev->attrs.max_qp_wr,

		max_recv_wr = min_t(int, wr_limit,

			      sess->queue_depth * 3 + 1);

		max_send_sge = sess->clt->max_segments + 1;

	}

drivers/infiniband/ulp/rtrs/rtrs-srv.c

+4 −9
Original line number Diff line number Diff line
@@ -1649,22 +1649,17 @@ static int create_con(struct rtrs_srv_sess *sess, 
	con->c.sess = &sess->s;

	con->c.cid = cid;

	atomic_set(&con->wr_cnt, 1);

	wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr;



	if (con->c.cid == 0) {

		/*

		 * All receive and all send (each requiring invalidate)

		 * + 2 for drain and heartbeat

		 */

		max_send_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;

		max_recv_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;

		max_send_wr = min_t(int, wr_limit,

				    SERVICE_CON_QUEUE_DEPTH * 2 + 2);

		max_recv_wr = max_send_wr;

	} else {

		/*

		 * In theory we might have queue_depth * 32

		 * outstanding requests if an unsafe global key is used

		 * and we have queue_depth read requests each consisting

		 * of 32 different addresses. div 3 for mlx5.

		 */

		wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr / 3;

		/* when always_invlaidate enalbed, we need linv+rinv+mr+imm */

		if (always_invalidate)

			max_send_wr =