Commit d796cea7 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

ublk: implement ->queue_rqs() 



Implement ->queue_rqs() for improving perf in case of MQ.

In this way, we just need to call io_uring_cmd_complete_in_task() once for
whole IO batch, then both io_uring and ublk server can get exact batch from
ublk frontend.

Follows IOPS improvement:

- tests

	tools/testing/selftests/ublk/kublk add -t null -q 2 [-z]

	fio/t/io_uring -p0 /dev/ublkb0

- results:

	more than 10% IOPS boost observed

Pass all ublk selftests, especially the io dispatch order test.

Cc: Uday Shankar <ushankar@purestorage.com>
Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250327095123.179113-9-ming.lei@redhat.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 17970209

drivers/block/ublk_drv.c

+111 −20
Original line number Diff line number Diff line
@@ -81,6 +81,20 @@ struct ublk_rq_data { 
};



struct ublk_uring_cmd_pdu {

	/*

	 * Store requests in same batch temporarily for queuing them to

	 * daemon context.

	 *

	 * It should have been stored to request payload, but we do want

	 * to avoid extra pre-allocation, and uring_cmd payload is always

	 * free for us

	 */

	struct request *req_list;



	/*

	 * The following two are valid in this cmd whole lifetime, and

	 * setup in ublk uring_cmd handler

	 */

	struct ublk_queue *ubq;

	u16 tag;

};
@@ -1170,14 +1184,12 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq, 
		blk_mq_end_request(rq, BLK_STS_IOERR);

}



static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd,

static void ublk_dispatch_req(struct ublk_queue *ubq,

			      struct io_uring_cmd *cmd,

			      struct request *req,

			      unsigned int issue_flags)

{

	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);

	struct ublk_queue *ubq = pdu->ubq;

	int tag = pdu->tag;

	struct request *req = blk_mq_tag_to_rq(

		ubq->dev->tag_set.tags[ubq->q_id], tag);

	int tag = req->tag;

	struct ublk_io *io = &ubq->ios[tag];

	unsigned int mapped_bytes;
@@ -1252,6 +1264,18 @@ static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd, 
	ubq_complete_io_cmd(io, UBLK_IO_RES_OK, issue_flags);

}



static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd,

				 unsigned int issue_flags)

{

	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);

	struct ublk_queue *ubq = pdu->ubq;

	int tag = pdu->tag;

	struct request *req = blk_mq_tag_to_rq(

		ubq->dev->tag_set.tags[ubq->q_id], tag);



	ublk_dispatch_req(ubq, cmd, req, issue_flags);

}



static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)

{

	struct ublk_io *io = &ubq->ios[rq->tag];
@@ -1259,6 +1283,35 @@ static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq) 
	io_uring_cmd_complete_in_task(io->cmd, ublk_rq_task_work_cb);

}



static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,

		unsigned int issue_flags)

{

	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);

	struct request *rq = pdu->req_list;

	struct ublk_queue *ubq = rq->mq_hctx->driver_data;

	struct request *next;



	while (rq) {

		struct ublk_io *io = &ubq->ios[rq->tag];



		next = rq->rq_next;

		rq->rq_next = NULL;

		ublk_dispatch_req(ubq, io->cmd, rq, issue_flags);

		rq = next;

	}

}



static void ublk_queue_cmd_list(struct ublk_queue *ubq, struct rq_list *l)

{

	struct request *rq = rq_list_peek(l);

	struct ublk_io *io = &ubq->ios[rq->tag];

	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(io->cmd);



	pdu->req_list = rq;

	rq_list_init(l);

	io_uring_cmd_complete_in_task(io->cmd, ublk_cmd_list_tw_cb);

}



static enum blk_eh_timer_return ublk_timeout(struct request *rq)

{

	struct ublk_queue *ubq = rq->mq_hctx->driver_data;
@@ -1297,21 +1350,12 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq) 
	return BLK_EH_RESET_TIMER;

}



static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,

		const struct blk_mq_queue_data *bd)

static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq)

{

	struct ublk_queue *ubq = hctx->driver_data;

	struct request *rq = bd->rq;

	blk_status_t res;



	if (unlikely(ubq->fail_io)) {

	if (unlikely(ubq->fail_io))

		return BLK_STS_TARGET;

	}



	/* fill iod to slot in io cmd buffer */

	res = ublk_setup_iod(ubq, rq);

	if (unlikely(res != BLK_STS_OK))

		return BLK_STS_IOERR;



	/* With recovery feature enabled, force_abort is set in

	 * ublk_stop_dev() before calling del_gendisk(). We have to
@@ -1325,6 +1369,29 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, 
	if (ublk_nosrv_should_queue_io(ubq) && unlikely(ubq->force_abort))

		return BLK_STS_IOERR;



	if (unlikely(ubq->canceling))

		return BLK_STS_IOERR;



	/* fill iod to slot in io cmd buffer */

	res = ublk_setup_iod(ubq, rq);

	if (unlikely(res != BLK_STS_OK))

		return BLK_STS_IOERR;



	blk_mq_start_request(rq);

	return BLK_STS_OK;

}



static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,

		const struct blk_mq_queue_data *bd)

{

	struct ublk_queue *ubq = hctx->driver_data;

	struct request *rq = bd->rq;

	blk_status_t res;



	res = ublk_prep_req(ubq, rq);

	if (res != BLK_STS_OK)

		return res;



	/*

	 * ->canceling has to be handled after ->force_abort and ->fail_io

	 * is dealt with, otherwise this request may not be failed in case
@@ -1335,12 +1402,35 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, 
		return BLK_STS_OK;

	}



	blk_mq_start_request(bd->rq);

	ublk_queue_cmd(ubq, rq);



	return BLK_STS_OK;

}



static void ublk_queue_rqs(struct rq_list *rqlist)

{

	struct rq_list requeue_list = { };

	struct rq_list submit_list = { };

	struct ublk_queue *ubq = NULL;

	struct request *req;



	while ((req = rq_list_pop(rqlist))) {

		struct ublk_queue *this_q = req->mq_hctx->driver_data;



		if (ubq && ubq != this_q && !rq_list_empty(&submit_list))

			ublk_queue_cmd_list(ubq, &submit_list);

		ubq = this_q;



		if (ublk_prep_req(ubq, req) == BLK_STS_OK)

			rq_list_add_tail(&submit_list, req);

		else

			rq_list_add_tail(&requeue_list, req);

	}



	if (ubq && !rq_list_empty(&submit_list))

		ublk_queue_cmd_list(ubq, &submit_list);

	*rqlist = requeue_list;

}



static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,

		unsigned int hctx_idx)

{
@@ -1353,6 +1443,7 @@ static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data, 


static const struct blk_mq_ops ublk_mq_ops = {

	.queue_rq       = ublk_queue_rq,

	.queue_rqs      = ublk_queue_rqs,

	.init_hctx	= ublk_init_hctx,

	.timeout	= ublk_timeout,

};