Commit 20fb3d05 authored by Caleb Sander Mateos's avatar Caleb Sander Mateos Committed by Jens Axboe
Browse files

io_uring/uring_cmd: avoid double indirect call in task work dispatch 



io_uring task work dispatch makes an indirect call to struct io_kiocb's
io_task_work.func field to allow running arbitrary task work functions.
In the uring_cmd case, this calls io_uring_cmd_work(), which immediately
makes another indirect call to struct io_uring_cmd's task_work_cb field.
Change the uring_cmd task work callbacks to functions whose signatures
match io_req_tw_func_t. Add a function io_uring_cmd_from_tw() to convert
from the task work's struct io_tw_req argument to struct io_uring_cmd *.
Define a constant IO_URING_CMD_TASK_WORK_ISSUE_FLAGS to avoid
manufacturing issue_flags in the uring_cmd task work callbacks. Now
uring_cmd task work dispatch makes a single indirect call to the
uring_cmd implementation's callback. This also allows removing the
task_work_cb field from struct io_uring_cmd, freeing up 8 bytes for
future storage.
Since fuse_uring_send_in_task() now has access to the io_tw_token_t,
check its cancel field directly instead of relying on the
IO_URING_F_TASK_DEAD issue flag.

Signed-off-by: default avatarCaleb Sander Mateos <csander@purestorage.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c33e779a

block/ioctl.c

+4 −2
Original line number Diff line number Diff line
@@ -769,14 +769,16 @@ struct blk_iou_cmd { 
	bool nowait;

};



static void blk_cmd_complete(struct io_uring_cmd *cmd, unsigned int issue_flags)

static void blk_cmd_complete(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);

	struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd);



	if (bic->res == -EAGAIN && bic->nowait)

		io_uring_cmd_issue_blocking(cmd);

	else

		io_uring_cmd_done(cmd, bic->res, issue_flags);

		io_uring_cmd_done(cmd, bic->res,

				  IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);

}



static void bio_cmd_bio_end_io(struct bio *bio)

drivers/block/ublk_drv.c

+11 −11
Original line number Diff line number Diff line
@@ -1302,10 +1302,9 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, 
	return true;

}



static void ublk_dispatch_req(struct ublk_queue *ubq,

			      struct request *req,

			      unsigned int issue_flags)

static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)

{

	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;

	int tag = req->tag;

	struct ublk_io *io = &ubq->ios[tag];
@@ -1348,13 +1347,13 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, 
		ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags);

}



static void ublk_cmd_tw_cb(struct io_uring_cmd *cmd,

			   unsigned int issue_flags)

static void ublk_cmd_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);

	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);

	struct ublk_queue *ubq = pdu->ubq;



	ublk_dispatch_req(ubq, pdu->req, issue_flags);

	ublk_dispatch_req(ubq, pdu->req);

}



static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
@@ -1366,9 +1365,9 @@ static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq) 
	io_uring_cmd_complete_in_task(cmd, ublk_cmd_tw_cb);

}



static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,

		unsigned int issue_flags)

static void ublk_cmd_list_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);

	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);

	struct request *rq = pdu->req_list;

	struct request *next;
@@ -1376,7 +1375,7 @@ static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd, 
	do {

		next = rq->rq_next;

		rq->rq_next = NULL;

		ublk_dispatch_req(rq->mq_hctx->driver_data, rq, issue_flags);

		ublk_dispatch_req(rq->mq_hctx->driver_data, rq);

		rq = next;

	} while (rq);

}
@@ -2523,9 +2522,10 @@ static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, 
	return NULL;

}



static void ublk_ch_uring_cmd_cb(struct io_uring_cmd *cmd,

		unsigned int issue_flags)

static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)

{

	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;

	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);

	int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);



	if (ret != -EIOCBQUEUED)

drivers/nvme/host/ioctl.c

+4 −3
Original line number Diff line number Diff line
@@ -398,14 +398,15 @@ static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu( 
	return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu);

}



static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,

			       unsigned issue_flags)

static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);

	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);



	if (pdu->bio)

		blk_rq_unmap_user(pdu->bio);

	io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, issue_flags);

	io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,

			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);

}



static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,

fs/btrfs/ioctl.c

+3 −2
Original line number Diff line number Diff line
@@ -4649,8 +4649,9 @@ struct io_btrfs_cmd { 
	struct btrfs_uring_priv *priv;

};



static void btrfs_uring_read_finished(struct io_uring_cmd *cmd, unsigned int issue_flags)

static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw)

{

	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);

	struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd);

	struct btrfs_uring_priv *priv = bc->priv;

	struct btrfs_inode *inode = BTRFS_I(file_inode(priv->iocb.ki_filp));
@@ -4695,7 +4696,7 @@ static void btrfs_uring_read_finished(struct io_uring_cmd *cmd, unsigned int iss 
	btrfs_unlock_extent(io_tree, priv->start, priv->lockend, &priv->cached_state);

	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);



	io_uring_cmd_done(cmd, ret, issue_flags);

	io_uring_cmd_done(cmd, ret, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);

	add_rchar(current, ret);



	for (index = 0; index < priv->nr_pages; index++)

fs/fuse/dev_uring.c

+4 −3
Original line number Diff line number Diff line
@@ -1209,14 +1209,15 @@ static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, 
 * User buffers are not mapped yet - the application does not have permission

 * to write to it - this has to be executed in ring task context.

 */

static void fuse_uring_send_in_task(struct io_uring_cmd *cmd,

				    unsigned int issue_flags)

static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw)

{

	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;

	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);

	struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);

	struct fuse_ring_queue *queue = ent->queue;

	int err;



	if (!(issue_flags & IO_URING_F_TASK_DEAD)) {

	if (!tw.cancel) {

		err = fuse_uring_prepare_send(ent, ent->fuse_req);

		if (err) {

			fuse_uring_next_fuse_req(ent, queue, issue_flags);