Commit 94ee8708 authored by Wilfred Mallawa's avatar Wilfred Mallawa Committed by Christoph Hellwig
Browse files

nvmet: support completion queue sharing 



The NVMe PCI transport specification allows for completion queues to be
shared by different submission queues.

This patch allows a submission queue to keep track of the completion queue
it is using with reference counting. As such, it can be ensured that a
completion queue is not deleted while a submission queue is actively
using it.

This patch enables completion queue sharing in the pci-epf target driver.
For fabrics drivers, completion queue sharing is not enabled as it is
not possible as per the fabrics specification. However, this patch
modifies the fabrics drivers to correctly integrate the new API that
supports completion queue sharing.

Signed-off-by: default avatarWilfred Mallawa <wilfred.mallawa@wdc.com>
Reviewed-by: default avatarChaitanya Kulkarni <kch@nvidia.com>
Reviewed-by: default avatarDamien Le Moal <dlemoal@kernel.org>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
parent bb78836b

drivers/nvme/target/admin-cmd.c

+10 −9
Original line number Diff line number Diff line
@@ -63,14 +63,9 @@ static void nvmet_execute_create_sq(struct nvmet_req *req) 
	if (status != NVME_SC_SUCCESS)

		goto complete;



	/*

	 * Note: The NVMe specification allows multiple SQs to use the same CQ.

	 * However, the target code does not really support that. So for now,

	 * prevent this and fail the command if sqid and cqid are different.

	 */

	if (!cqid || cqid != sqid) {

		pr_err("SQ %u: Unsupported CQID %u\n", sqid, cqid);

		status = NVME_SC_CQ_INVALID | NVME_STATUS_DNR;

	status = nvmet_check_io_cqid(ctrl, cqid, false);

	if (status != NVME_SC_SUCCESS) {

		pr_err("SQ %u: Invalid CQID %u\n", sqid, cqid);

		goto complete;

	}
@@ -79,7 +74,7 @@ static void nvmet_execute_create_sq(struct nvmet_req *req) 
		goto complete;

	}



	status = ctrl->ops->create_sq(ctrl, sqid, sq_flags, qsize, prp1);

	status = ctrl->ops->create_sq(ctrl, sqid, cqid, sq_flags, qsize, prp1);



complete:

	nvmet_req_complete(req, status);
@@ -100,6 +95,12 @@ static void nvmet_execute_delete_cq(struct nvmet_req *req) 
	if (status != NVME_SC_SUCCESS)

		goto complete;



	if (!ctrl->cqs[cqid] || nvmet_cq_in_use(ctrl->cqs[cqid])) {

		/* Some SQs are still using this CQ */

		status = NVME_SC_QID_INVALID | NVME_STATUS_DNR;

		goto complete;

	}



	status = ctrl->ops->delete_cq(ctrl, cqid);



complete:

drivers/nvme/target/core.c

+14 −3
Original line number Diff line number Diff line
@@ -905,6 +905,11 @@ u16 nvmet_cq_create(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, 
	if (status != NVME_SC_SUCCESS)

		return status;



	if (!kref_get_unless_zero(&ctrl->ref))

		return NVME_SC_INTERNAL | NVME_STATUS_DNR;

	cq->ctrl = ctrl;



	nvmet_cq_init(cq);

	nvmet_cq_setup(ctrl, cq, qid, size);



	return NVME_SC_SUCCESS;
@@ -928,7 +933,7 @@ u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, 
}



u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,

		    u16 sqid, u16 size)

		    struct nvmet_cq *cq, u16 sqid, u16 size)

{

	u16 status;

	int ret;
@@ -940,7 +945,7 @@ u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, 
	if (status != NVME_SC_SUCCESS)

		return status;



	ret = nvmet_sq_init(sq);

	ret = nvmet_sq_init(sq, cq);

	if (ret) {

		status = NVME_SC_INTERNAL | NVME_STATUS_DNR;

		goto ctrl_put;
@@ -972,6 +977,7 @@ void nvmet_sq_destroy(struct nvmet_sq *sq) 
	wait_for_completion(&sq->free_done);

	percpu_ref_exit(&sq->ref);

	nvmet_auth_sq_free(sq);

	nvmet_cq_put(sq->cq);



	/*

	 * we must reference the ctrl again after waiting for inflight IO
@@ -1004,18 +1010,23 @@ static void nvmet_sq_free(struct percpu_ref *ref) 
	complete(&sq->free_done);

}



int nvmet_sq_init(struct nvmet_sq *sq)

int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq)

{

	int ret;



	if (!nvmet_cq_get(cq))

		return -EINVAL;



	ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL);

	if (ret) {

		pr_err("percpu_ref init failed!\n");

		nvmet_cq_put(cq);

		return ret;

	}

	init_completion(&sq->free_done);

	init_completion(&sq->confirm_done);

	nvmet_auth_sq_init(sq);

	sq->cq = cq;



	return 0;

}

drivers/nvme/target/fc.c

+1 −1
Original line number Diff line number Diff line
@@ -817,7 +817,7 @@ nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, 
	nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);



	nvmet_cq_init(&queue->nvme_cq);

	ret = nvmet_sq_init(&queue->nvme_sq);

	ret = nvmet_sq_init(&queue->nvme_sq, &queue->nvme_cq);

	if (ret)

		goto out_fail_iodlist;

drivers/nvme/target/loop.c

+4 −2
Original line number Diff line number Diff line
@@ -332,7 +332,8 @@ static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl) 
	for (i = 1; i <= nr_io_queues; i++) {

		ctrl->queues[i].ctrl = ctrl;

		nvmet_cq_init(&ctrl->queues[i].nvme_cq);

		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);

		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq,

				&ctrl->queues[i].nvme_cq);

		if (ret) {

			nvmet_cq_put(&ctrl->queues[i].nvme_cq);

			goto out_destroy_queues;
@@ -368,7 +369,8 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) 


	ctrl->queues[0].ctrl = ctrl;

	nvmet_cq_init(&ctrl->queues[0].nvme_cq);

	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);

	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq,

			&ctrl->queues[0].nvme_cq);

	if (error) {

		nvmet_cq_put(&ctrl->queues[0].nvme_cq);

		return error;

drivers/nvme/target/nvmet.h

+5 −4
Original line number Diff line number Diff line
@@ -150,6 +150,7 @@ struct nvmet_cq { 
struct nvmet_sq {

	struct nvmet_ctrl	*ctrl;

	struct percpu_ref	ref;

	struct nvmet_cq		*cq;

	u16			qid;

	u16			size;

	u32			sqhd;
@@ -427,7 +428,7 @@ struct nvmet_fabrics_ops { 
	u16 (*get_max_queue_size)(const struct nvmet_ctrl *ctrl);



	/* Operations mandatory for PCI target controllers */

	u16 (*create_sq)(struct nvmet_ctrl *ctrl, u16 sqid, u16 flags,

	u16 (*create_sq)(struct nvmet_ctrl *ctrl, u16 sqid, u16 cqid, u16 flags,

			 u16 qsize, u64 prp1);

	u16 (*delete_sq)(struct nvmet_ctrl *ctrl, u16 sqid);

	u16 (*create_cq)(struct nvmet_ctrl *ctrl, u16 cqid, u16 flags,
@@ -588,10 +589,10 @@ bool nvmet_cq_in_use(struct nvmet_cq *cq); 
u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, bool create);

void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, u16 qid,

		u16 size);

u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, u16 qid,

		u16 size);

u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,

	struct nvmet_cq *cq, u16 qid, u16 size);

void nvmet_sq_destroy(struct nvmet_sq *sq);

int nvmet_sq_init(struct nvmet_sq *sq);

int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq);



void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl);