Commit 0e13dd9e authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: Remove temp allocation of rpcrdma_rep objects 



The original code was designed so that most calls to
rpcrdma_rep_create() would occur on the NUMA node that the device
preferred. There are a few cases where that's not possible, so
those reps are marked as temporary.

However, we have the device (and its preferred node) already in
rpcrdma_rep_create(), so let's use that to guarantee the memory
is allocated from the correct node.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Reviewed-by: default avatarSagi Grimberg <sagi@grimberg.me>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 9d53378c

net/sunrpc/xprtrdma/rpc_rdma.c

+1 −2
Original line number Diff line number Diff line
@@ -1471,8 +1471,7 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) 
		credits = 1;	/* don't deadlock */

	else if (credits > r_xprt->rx_ep->re_max_requests)

		credits = r_xprt->rx_ep->re_max_requests;

	rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1),

			   false);

	rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1));

	if (buf->rb_credits != credits)

		rpcrdma_update_cwnd(r_xprt, credits);

net/sunrpc/xprtrdma/verbs.c

+24 −33
Original line number Diff line number Diff line
@@ -69,13 +69,15 @@ static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, 
				       struct rpcrdma_sendctx *sc);

static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);

static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);

static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep);

static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);

static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);

static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);

static void rpcrdma_ep_get(struct rpcrdma_ep *ep);

static int rpcrdma_ep_put(struct rpcrdma_ep *ep);

static struct rpcrdma_regbuf *

rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction,

			  int node);

static struct rpcrdma_regbuf *

rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction);

static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);

static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
@@ -510,7 +512,7 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) 
	 * outstanding Receives.

	 */

	rpcrdma_ep_get(ep);

	rpcrdma_post_recvs(r_xprt, 1, true);

	rpcrdma_post_recvs(r_xprt, 1);



	rc = rdma_connect(ep->re_id, &ep->re_remote_cma);

	if (rc)
@@ -943,18 +945,20 @@ static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt) 
}



static noinline

struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,

				       bool temp)

struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt)

{

	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;

	struct rpcrdma_ep *ep = r_xprt->rx_ep;

	struct ib_device *device = ep->re_id->device;

	struct rpcrdma_rep *rep;



	rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS);

	if (rep == NULL)

		goto out;



	rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,

					       DMA_FROM_DEVICE);

	rep->rr_rdmabuf = rpcrdma_regbuf_alloc_node(ep->re_inline_recv,

						    DMA_FROM_DEVICE,

						    ibdev_to_node(device));

	if (!rep->rr_rdmabuf)

		goto out_free;
@@ -969,7 +973,6 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, 
	rep->rr_recv_wr.wr_cqe = &rep->rr_cqe;

	rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;

	rep->rr_recv_wr.num_sge = 1;

	rep->rr_temp = temp;



	spin_lock(&buf->rb_lock);

	list_add(&rep->rr_all, &buf->rb_all_reps);
@@ -988,17 +991,6 @@ static void rpcrdma_rep_free(struct rpcrdma_rep *rep) 
	kfree(rep);

}



static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep)

{

	struct rpcrdma_buffer *buf = &rep->rr_rxprt->rx_buf;



	spin_lock(&buf->rb_lock);

	list_del(&rep->rr_all);

	spin_unlock(&buf->rb_lock);



	rpcrdma_rep_free(rep);

}



static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf)

{

	struct llist_node *node;
@@ -1030,10 +1022,8 @@ static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt) 
	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;

	struct rpcrdma_rep *rep;



	list_for_each_entry(rep, &buf->rb_all_reps, rr_all) {

	list_for_each_entry(rep, &buf->rb_all_reps, rr_all)

		rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf);

		rep->rr_temp = true;	/* Mark this rep for destruction */

	}

}



static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf)
@@ -1250,14 +1240,15 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) 
 * or Replies they may be registered externally via frwr_map.

 */

static struct rpcrdma_regbuf *

rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)

rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction,

			  int node)

{

	struct rpcrdma_regbuf *rb;



	rb = kmalloc(sizeof(*rb), XPRTRDMA_GFP_FLAGS);

	rb = kmalloc_node(sizeof(*rb), XPRTRDMA_GFP_FLAGS, node);

	if (!rb)

		return NULL;

	rb->rg_data = kmalloc(size, XPRTRDMA_GFP_FLAGS);

	rb->rg_data = kmalloc_node(size, XPRTRDMA_GFP_FLAGS, node);

	if (!rb->rg_data) {

		kfree(rb);

		return NULL;
@@ -1269,6 +1260,12 @@ rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) 
	return rb;

}



static struct rpcrdma_regbuf *

rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)

{

	return rpcrdma_regbuf_alloc_node(size, direction, NUMA_NO_NODE);

}



/**

 * rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer

 * @rb: regbuf to reallocate
@@ -1346,10 +1343,9 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb) 
 * rpcrdma_post_recvs - Refill the Receive Queue

 * @r_xprt: controlling transport instance

 * @needed: current credit grant

 * @temp: mark Receive buffers to be deleted after one use

 *

 */

void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)

void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed)

{

	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;

	struct rpcrdma_ep *ep = r_xprt->rx_ep;
@@ -1363,7 +1359,6 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp) 
	if (likely(ep->re_receive_count > needed))

		goto out;

	needed -= ep->re_receive_count;

	if (!temp)

	needed += RPCRDMA_MAX_RECV_BATCH;



	if (atomic_inc_return(&ep->re_receiving) > 1)
@@ -1373,12 +1368,8 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp) 
	wr = NULL;

	while (needed) {

		rep = rpcrdma_rep_get_locked(buf);

		if (rep && rep->rr_temp) {

			rpcrdma_rep_destroy(rep);

			continue;

		}

		if (!rep)

			rep = rpcrdma_rep_create(r_xprt, temp);

			rep = rpcrdma_rep_create(r_xprt);

		if (!rep)

			break;

		if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) {

net/sunrpc/xprtrdma/xprt_rdma.h

+1 −2
Original line number Diff line number Diff line
@@ -200,7 +200,6 @@ struct rpcrdma_rep { 
	__be32			rr_proc;

	int			rr_wc_flags;

	u32			rr_inv_rkey;

	bool			rr_temp;

	struct rpcrdma_regbuf	*rr_rdmabuf;

	struct rpcrdma_xprt	*rr_rxprt;

	struct rpc_rqst		*rr_rqst;
@@ -468,7 +467,7 @@ void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc); 
int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt);

void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt);



void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp);

void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed);



/*

 * Buffer calls - xprtrdma/verbs.c