Commit bf098d72 authored by Uday Shankar's avatar Uday Shankar Committed by Jens Axboe
Browse files

selftests: ublk: kublk: plumb q_id in io_uring user_data 



Currently, when we process CQEs, we know which ublk_queue we are working
on because we know which ring we are working on, and ublk_queues and
rings are in 1:1 correspondence. However, as we decouple ublk_queues
from ublk server threads, ublk_queues and rings will no longer be in 1:1
correspondence - each ublk server thread will have a ring, and each
thread may issue commands against more than one ublk_queue. So in order
to know which ublk_queue a CQE refers to, plumb that information in the
associated SQE's user_data.

Signed-off-by: default avatarUday Shankar <ushankar@purestorage.com>
Reviewed-by: default avatarMing Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250529-ublk_task_per_io-v8-2-e9d3b119336a@purestorage.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent ab03a61c

tools/testing/selftests/ublk/fault_inject.c

+1 −1
Original line number Diff line number Diff line
@@ -48,7 +48,7 @@ static int ublk_fault_inject_queue_io(struct ublk_queue *q, int tag) 


	ublk_queue_alloc_sqes(q, &sqe, 1);

	io_uring_prep_timeout(sqe, &ts, 1, 0);

	sqe->user_data = build_user_data(tag, ublksrv_get_op(iod), 0, 1);

	sqe->user_data = build_user_data(tag, ublksrv_get_op(iod), 0, q->q_id, 1);



	ublk_queued_tgt_io(q, tag, 1);

tools/testing/selftests/ublk/file_backed.c

+5 −5
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ static int loop_queue_flush_io(struct ublk_queue *q, const struct ublksrv_io_des 
	io_uring_prep_fsync(sqe[0], 1 /*fds[1]*/, IORING_FSYNC_DATASYNC);

	io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);

	/* bit63 marks us as tgt io */

	sqe[0]->user_data = build_user_data(tag, ublk_op, 0, 1);

	sqe[0]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);

	return 1;

}
@@ -48,7 +48,7 @@ static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_de 
			sqe[0]->buf_index = tag;

		io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);

		/* bit63 marks us as tgt io */

		sqe[0]->user_data = build_user_data(tag, ublk_op, 0, 1);

		sqe[0]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);

		return 1;

	}
@@ -57,17 +57,17 @@ static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_de 
	io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, tag);

	sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;

	sqe[0]->user_data = build_user_data(tag,

			ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);

			ublk_cmd_op_nr(sqe[0]->cmd_op), 0, q->q_id, 1);



	io_uring_prep_rw(op, sqe[1], 1 /*fds[1]*/, 0,

		iod->nr_sectors << 9,

		iod->start_sector << 9);

	sqe[1]->buf_index = tag;

	sqe[1]->flags |= IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK;

	sqe[1]->user_data = build_user_data(tag, ublk_op, 0, 1);

	sqe[1]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);



	io_uring_prep_buf_unregister(sqe[2], 0, tag, q->q_id, tag);

	sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, 1);

	sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, q->q_id, 1);



	return 2;

}

tools/testing/selftests/ublk/kublk.c

+9 −8
Original line number Diff line number Diff line
@@ -627,7 +627,7 @@ int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag) 
	if (q->state & UBLKSRV_AUTO_BUF_REG)

		ublk_set_auto_buf_reg(q, sqe[0], tag);



	user_data = build_user_data(tag, _IOC_NR(cmd_op), 0, 0);

	user_data = build_user_data(tag, _IOC_NR(cmd_op), 0, q->q_id, 0);

	io_uring_sqe_set_data64(sqe[0], user_data);



	io->flags = 0;
@@ -673,10 +673,11 @@ static inline void ublksrv_handle_tgt_cqe(struct ublk_queue *q, 
		q->tgt_ops->tgt_io_done(q, tag, cqe);

}



static void ublk_handle_cqe(struct io_uring *r,

static void ublk_handle_cqe(struct ublk_dev *dev,

		struct io_uring_cqe *cqe, void *data)

{

	struct ublk_queue *q = container_of(r, struct ublk_queue, ring);

	unsigned q_id = user_data_to_q_id(cqe->user_data);

	struct ublk_queue *q = &dev->q[q_id];

	unsigned tag = user_data_to_tag(cqe->user_data);

	unsigned cmd_op = user_data_to_op(cqe->user_data);

	int fetch = (cqe->res != UBLK_IO_RES_ABORT) &&
@@ -727,17 +728,17 @@ static void ublk_handle_cqe(struct io_uring *r, 
	}

}



static int ublk_reap_events_uring(struct io_uring *r)

static int ublk_reap_events_uring(struct ublk_queue *q)

{

	struct io_uring_cqe *cqe;

	unsigned head;

	int count = 0;



	io_uring_for_each_cqe(r, head, cqe) {

		ublk_handle_cqe(r, cqe, NULL);

	io_uring_for_each_cqe(&q->ring, head, cqe) {

		ublk_handle_cqe(q->dev, cqe, NULL);

		count += 1;

	}

	io_uring_cq_advance(r, count);

	io_uring_cq_advance(&q->ring, count);



	return count;

}
@@ -756,7 +757,7 @@ static int ublk_process_io(struct ublk_queue *q) 
		return -ENODEV;



	ret = io_uring_submit_and_wait(&q->ring, 1);

	reapped = ublk_reap_events_uring(&q->ring);

	reapped = ublk_reap_events_uring(q);



	ublk_dbg(UBLK_DBG_QUEUE, "submit result %d, reapped %d stop %d idle %d\n",

			ret, reapped, (q->state & UBLKSRV_QUEUE_STOPPING),

tools/testing/selftests/ublk/kublk.h

+13 −4
Original line number Diff line number Diff line
@@ -49,7 +49,8 @@ 
#define UBLKSRV_IO_IDLE_SECS		20



#define UBLK_IO_MAX_BYTES               (1 << 20)

#define UBLK_MAX_QUEUES                 32

#define UBLK_MAX_QUEUES_SHIFT		5

#define UBLK_MAX_QUEUES                 (1 << UBLK_MAX_QUEUES_SHIFT)

#define UBLK_QUEUE_DEPTH                1024



#define UBLK_DBG_DEV            (1U << 0)
@@ -225,11 +226,14 @@ static inline int is_target_io(__u64 user_data) 
}



static inline __u64 build_user_data(unsigned tag, unsigned op,

		unsigned tgt_data, unsigned is_target_io)

		unsigned tgt_data, unsigned q_id, unsigned is_target_io)

{

	assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16));

	/* we only have 7 bits to encode q_id */

	_Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7);

	assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16) && !(q_id >> 7));



	return tag | (op << 16) | (tgt_data << 24) | (__u64)is_target_io << 63;

	return tag | (op << 16) | (tgt_data << 24) |

		(__u64)q_id << 56 | (__u64)is_target_io << 63;

}



static inline unsigned int user_data_to_tag(__u64 user_data)
@@ -247,6 +251,11 @@ static inline unsigned int user_data_to_tgt_data(__u64 user_data) 
	return (user_data >> 24) & 0xffff;

}



static inline unsigned int user_data_to_q_id(__u64 user_data)

{

	return (user_data >> 56) & 0x7f;

}



static inline unsigned short ublk_cmd_op_nr(unsigned int op)

{

	return _IOC_NR(op);

tools/testing/selftests/ublk/null.c

+6 −6
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ static int ublk_null_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev) 
}



static void __setup_nop_io(int tag, const struct ublksrv_io_desc *iod,

		struct io_uring_sqe *sqe)

		struct io_uring_sqe *sqe, int q_id)

{

	unsigned ublk_op = ublksrv_get_op(iod);
@@ -52,7 +52,7 @@ static void __setup_nop_io(int tag, const struct ublksrv_io_desc *iod, 
	sqe->flags |= IOSQE_FIXED_FILE;

	sqe->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;

	sqe->len = iod->nr_sectors << 9; 	/* injected result */

	sqe->user_data = build_user_data(tag, ublk_op, 0, 1);

	sqe->user_data = build_user_data(tag, ublk_op, 0, q_id, 1);

}



static int null_queue_zc_io(struct ublk_queue *q, int tag)
@@ -64,14 +64,14 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag) 


	io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, tag);

	sqe[0]->user_data = build_user_data(tag,

			ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);

			ublk_cmd_op_nr(sqe[0]->cmd_op), 0, q->q_id, 1);

	sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;



	__setup_nop_io(tag, iod, sqe[1]);

	__setup_nop_io(tag, iod, sqe[1], q->q_id);

	sqe[1]->flags |= IOSQE_IO_HARDLINK;



	io_uring_prep_buf_unregister(sqe[2], 0, tag, q->q_id, tag);

	sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, 1);

	sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, q->q_id, 1);



	// buf register is marked as IOSQE_CQE_SKIP_SUCCESS

	return 2;
@@ -83,7 +83,7 @@ static int null_queue_auto_zc_io(struct ublk_queue *q, int tag) 
	struct io_uring_sqe *sqe[1];



	ublk_queue_alloc_sqes(q, sqe, 1);

	__setup_nop_io(tag, iod, sqe[0]);

	__setup_nop_io(tag, iod, sqe[0], q->q_id);

	return 1;

}