Commit 3859e39d authored by Ralph Campbell's avatar Ralph Campbell Committed by Roland Dreier
Browse files

IB/ipath: Support larger IB_QP_MAX_DEST_RD_ATOMIC and IB_QP_MAX_QP_RD_ATOMIC 



This patch adds support for multiple RDMA reads and atomics to be sent
before an ACK is required to be seen by the requester.

Signed-off-by: default avatarBryan O'Sullivan <bryan.osullivan@qlogic.com>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent 7b21d26d

drivers/infiniband/hw/ipath/ipath_qp.c

+20 −6
Original line number Diff line number Diff line
@@ -320,7 +320,8 @@ static void ipath_reset_qp(struct ipath_qp *qp) 
	qp->remote_qpn = 0;

	qp->qkey = 0;

	qp->qp_access_flags = 0;

	clear_bit(IPATH_S_BUSY, &qp->s_flags);

	qp->s_busy = 0;

	qp->s_flags &= ~IPATH_S_SIGNAL_REQ_WR;

	qp->s_hdrwords = 0;

	qp->s_psn = 0;

	qp->r_psn = 0;
@@ -333,7 +334,6 @@ static void ipath_reset_qp(struct ipath_qp *qp) 
		qp->r_state = IB_OPCODE_UC_SEND_LAST;

	}

	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;

	qp->r_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;

	qp->r_nak_state = 0;

	qp->r_wrid_valid = 0;

	qp->s_rnr_timeout = 0;
@@ -344,6 +344,10 @@ static void ipath_reset_qp(struct ipath_qp *qp) 
	qp->s_ssn = 1;

	qp->s_lsn = 0;

	qp->s_wait_credit = 0;

	memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));

	qp->r_head_ack_queue = 0;

	qp->s_tail_ack_queue = 0;

	qp->s_num_rd_atomic = 0;

	if (qp->r_rq.wq) {

		qp->r_rq.wq->head = 0;

		qp->r_rq.wq->tail = 0;
@@ -503,6 +507,10 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
		    attr->path_mig_state != IB_MIG_REARM)

			goto inval;



	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)

		if (attr->max_dest_rd_atomic > IPATH_MAX_RDMA_ATOMIC)

			goto inval;



	switch (new_state) {

	case IB_QPS_RESET:

		ipath_reset_qp(qp);
@@ -559,6 +567,12 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	if (attr_mask & IB_QP_QKEY)

		qp->qkey = attr->qkey;



	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)

		qp->r_max_rd_atomic = attr->max_dest_rd_atomic;



	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)

		qp->s_max_rd_atomic = attr->max_rd_atomic;



	qp->state = new_state;

	spin_unlock_irqrestore(&qp->s_lock, flags);
@@ -598,8 +612,8 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	attr->alt_pkey_index = 0;

	attr->en_sqd_async_notify = 0;

	attr->sq_draining = 0;

	attr->max_rd_atomic = 1;

	attr->max_dest_rd_atomic = 1;

	attr->max_rd_atomic = qp->s_max_rd_atomic;

	attr->max_dest_rd_atomic = qp->r_max_rd_atomic;

	attr->min_rnr_timer = qp->r_min_rnr_timer;

	attr->port_num = 1;

	attr->timeout = qp->timeout;
@@ -614,7 +628,7 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
	init_attr->recv_cq = qp->ibqp.recv_cq;

	init_attr->srq = qp->ibqp.srq;

	init_attr->cap = attr->cap;

	if (qp->s_flags & (1 << IPATH_S_SIGNAL_REQ_WR))

	if (qp->s_flags & IPATH_S_SIGNAL_REQ_WR)

		init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;

	else

		init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
@@ -786,7 +800,7 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd, 
		qp->s_size = init_attr->cap.max_send_wr + 1;

		qp->s_max_sge = init_attr->cap.max_send_sge;

		if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)

			qp->s_flags = 1 << IPATH_S_SIGNAL_REQ_WR;

			qp->s_flags = IPATH_S_SIGNAL_REQ_WR;

		else

			qp->s_flags = 0;

		dev = to_idev(ibpd->device);

drivers/infiniband/hw/ipath/ipath_rc.c

+454 −346

File changed.

Preview size limit exceeded, changes collapsed.

drivers/infiniband/hw/ipath/ipath_ruc.c

+30 −28
Original line number Diff line number Diff line
@@ -255,6 +255,7 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp) 
	unsigned long flags;

	struct ib_wc wc;

	u64 sdata;

	atomic64_t *maddr;



	qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn);

	if (!qp) {
@@ -311,7 +312,7 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp) 
				sqp->s_rnr_retry--;

			dev->n_rnr_naks++;

			sqp->s_rnr_timeout =

				ib_ipath_rnr_table[sqp->r_min_rnr_timer];

				ib_ipath_rnr_table[qp->r_min_rnr_timer];

			ipath_insert_rnr_queue(sqp);

			goto done;

		}
@@ -344,20 +345,22 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp) 
			wc.sl = sqp->remote_ah_attr.sl;

			wc.dlid_path_bits = 0;

			wc.port_num = 0;

			spin_lock_irqsave(&sqp->s_lock, flags);

			ipath_sqerror_qp(sqp, &wc);

			spin_unlock_irqrestore(&sqp->s_lock, flags);

			goto done;

		}

		break;



	case IB_WR_RDMA_READ:

		if (unlikely(!(qp->qp_access_flags &

			       IB_ACCESS_REMOTE_READ)))

			goto acc_err;

		if (unlikely(!ipath_rkey_ok(qp, &sqp->s_sge, wqe->length,

					    wqe->wr.wr.rdma.remote_addr,

					    wqe->wr.wr.rdma.rkey,

					    IB_ACCESS_REMOTE_READ)))

			goto acc_err;

		if (unlikely(!(qp->qp_access_flags &

			       IB_ACCESS_REMOTE_READ)))

			goto acc_err;

		qp->r_sge.sge = wqe->sg_list[0];

		qp->r_sge.sg_list = wqe->sg_list + 1;

		qp->r_sge.num_sge = wqe->wr.num_sge;
@@ -365,22 +368,22 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp) 


	case IB_WR_ATOMIC_CMP_AND_SWP:

	case IB_WR_ATOMIC_FETCH_AND_ADD:

		if (unlikely(!(qp->qp_access_flags &

			       IB_ACCESS_REMOTE_ATOMIC)))

			goto acc_err;

		if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, sizeof(u64),

					    wqe->wr.wr.rdma.remote_addr,

					    wqe->wr.wr.rdma.rkey,

					    wqe->wr.wr.atomic.remote_addr,

					    wqe->wr.wr.atomic.rkey,

					    IB_ACCESS_REMOTE_ATOMIC)))

			goto acc_err;

		/* Perform atomic OP and save result. */

		sdata = wqe->wr.wr.atomic.swap;

		spin_lock_irqsave(&dev->pending_lock, flags);

		qp->r_atomic_data = *(u64 *) qp->r_sge.sge.vaddr;

		if (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)

			*(u64 *) qp->r_sge.sge.vaddr =

				qp->r_atomic_data + sdata;

		else if (qp->r_atomic_data == wqe->wr.wr.atomic.compare_add)

			*(u64 *) qp->r_sge.sge.vaddr = sdata;

		spin_unlock_irqrestore(&dev->pending_lock, flags);

		*(u64 *) sqp->s_sge.sge.vaddr = qp->r_atomic_data;

		maddr = (atomic64_t *) qp->r_sge.sge.vaddr;

		sdata = wqe->wr.wr.atomic.compare_add;

		*(u64 *) sqp->s_sge.sge.vaddr =

			(wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?

			(u64) atomic64_add_return(sdata, maddr) - sdata :

			(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,

				      sdata, wqe->wr.wr.atomic.swap);

		goto send_comp;



	default:
@@ -441,7 +444,7 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp) 
send_comp:

	sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;



	if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &sqp->s_flags) ||

	if (!(sqp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||

	    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {

		wc.wr_id = wqe->wr.wr_id;

		wc.status = IB_WC_SUCCESS;
@@ -503,7 +506,7 @@ void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev) 
	 * We clear the tasklet flag now since we are committing to return

	 * from the tasklet function.

	 */

	clear_bit(IPATH_S_BUSY, &qp->s_flags);

	clear_bit(IPATH_S_BUSY, &qp->s_busy);

	tasklet_unlock(&qp->s_task);

	want_buffer(dev->dd);

	dev->n_piowait++;
@@ -542,6 +545,9 @@ int ipath_post_ruc_send(struct ipath_qp *qp, struct ib_send_wr *wr) 
		    wr->sg_list[0].addr & (sizeof(u64) - 1))) {

		ret = -EINVAL;

		goto bail;

	} else if (wr->opcode >= IB_WR_RDMA_READ && !qp->s_max_rd_atomic) {

		ret = -EINVAL;

		goto bail;

	}

	/* IB spec says that num_sge == 0 is OK. */

	if (wr->num_sge > qp->s_max_sge) {
@@ -648,7 +654,7 @@ void ipath_do_ruc_send(unsigned long data) 
	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);

	struct ipath_other_headers *ohdr;



	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))

	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_busy))

		goto bail;



	if (unlikely(qp->remote_ah_attr.dlid == dev->dd->ipath_lid)) {
@@ -684,11 +690,7 @@ void ipath_do_ruc_send(unsigned long data) 
	 */

	spin_lock_irqsave(&qp->s_lock, flags);



	/* Sending responses has higher priority over sending requests. */

	if (qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE &&

	    (bth0 = ipath_make_rc_ack(qp, ohdr, pmtu)) != 0)

		bth2 = qp->s_ack_psn++ & IPATH_PSN_MASK;

	else if (!((qp->ibqp.qp_type == IB_QPT_RC) ?

	if (!((qp->ibqp.qp_type == IB_QPT_RC) ?

	       ipath_make_rc_req(qp, ohdr, pmtu, &bth0, &bth2) :

	       ipath_make_uc_req(qp, ohdr, pmtu, &bth0, &bth2))) {

		/*
@@ -696,7 +698,7 @@ void ipath_do_ruc_send(unsigned long data) 
		 * adding new work queue items and then failing to process

		 * them.

		 */

		clear_bit(IPATH_S_BUSY, &qp->s_flags);

		clear_bit(IPATH_S_BUSY, &qp->s_busy);

		spin_unlock_irqrestore(&qp->s_lock, flags);

		goto bail;

	}
@@ -729,7 +731,7 @@ void ipath_do_ruc_send(unsigned long data) 
	goto again;



clear:

	clear_bit(IPATH_S_BUSY, &qp->s_flags);

	clear_bit(IPATH_S_BUSY, &qp->s_busy);

bail:

	return;

}

drivers/infiniband/hw/ipath/ipath_uc.c

+3 −3
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ static void complete_last_send(struct ipath_qp *qp, struct ipath_swqe *wqe, 
{

	if (++qp->s_last == qp->s_size)

		qp->s_last = 0;

	if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||

	if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||

	    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {

		wc->wr_id = wqe->wr.wr_id;

		wc->status = IB_WC_SUCCESS;
@@ -344,13 +344,13 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr, 
	send_first:

		if (qp->r_reuse_sge) {

			qp->r_reuse_sge = 0;

			qp->r_sge = qp->s_rdma_sge;

			qp->r_sge = qp->s_rdma_read_sge;

		} else if (!ipath_get_rwqe(qp, 0)) {

			dev->n_pkt_drops++;

			goto done;

		}

		/* Save the WQE so we can reuse it in case of an error. */

		qp->s_rdma_sge = qp->r_sge;

		qp->s_rdma_read_sge = qp->r_sge;

		qp->r_rcv_len = 0;

		if (opcode == OP(SEND_ONLY))

			goto send_last;

drivers/infiniband/hw/ipath/ipath_ud.c

+1 −1
Original line number Diff line number Diff line
@@ -467,7 +467,7 @@ int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr) 


done:

	/* Queue the completion status entry. */

	if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||

	if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||

	    (wr->send_flags & IB_SEND_SIGNALED)) {

		wc.wr_id = wr->wr_id;

		wc.status = IB_WC_SUCCESS;