Commit 8ccebc19 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe
Browse files

selftests: ublk: support UBLK_F_AUTO_BUF_REG 



Enable UBLK_F_AUTO_BUF_REG support for ublk utility by argument `--auto_zc`,
meantime support this feature in null, loop and stripe target code.

Add function test generic_08 for covering basic UBLK_F_AUTO_BUF_REG feature.

Also cover UBLK_F_AUTO_BUF_REG in stress_03, stress_04 and stress_05 test too.

'fio/t/io_uring -p0 /dev/ublkb0' shows that F_AUTO_BUF_REG can improve
IOPS by 50% compared with F_SUPPORT_ZERO_COPY in my test VM.

Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250520045455.515691-6-ming.lei@redhat.com


Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 53f427e7

tools/testing/selftests/ublk/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ TEST_PROGS += test_generic_05.sh 
TEST_PROGS += test_generic_06.sh

TEST_PROGS += test_generic_07.sh



TEST_PROGS += test_generic_08.sh



TEST_PROGS += test_null_01.sh

TEST_PROGS += test_null_02.sh

TEST_PROGS += test_loop_01.sh

tools/testing/selftests/ublk/file_backed.c

+8 −4
Original line number Diff line number Diff line
@@ -29,19 +29,23 @@ static int loop_queue_flush_io(struct ublk_queue *q, const struct ublksrv_io_des 
static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)

{

	unsigned ublk_op = ublksrv_get_op(iod);

	int zc = ublk_queue_use_zc(q);

	enum io_uring_op op = ublk_to_uring_op(iod, zc);

	unsigned zc = ublk_queue_use_zc(q);

	unsigned auto_zc = ublk_queue_use_auto_zc(q);

	enum io_uring_op op = ublk_to_uring_op(iod, zc | auto_zc);

	struct io_uring_sqe *sqe[3];

	void *addr = (zc | auto_zc) ? NULL : (void *)iod->addr;



	if (!zc) {

	if (!zc || auto_zc) {

		ublk_queue_alloc_sqes(q, sqe, 1);

		if (!sqe[0])

			return -ENOMEM;



		io_uring_prep_rw(op, sqe[0], 1 /*fds[1]*/,

				(void *)iod->addr,

				addr,

				iod->nr_sectors << 9,

				iod->start_sector << 9);

		if (auto_zc)

			sqe[0]->buf_index = tag;

		io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);

		/* bit63 marks us as tgt io */

		sqe[0]->user_data = build_user_data(tag, ublk_op, 0, 1);

tools/testing/selftests/ublk/kublk.c

+26 −4
Original line number Diff line number Diff line
@@ -420,9 +420,12 @@ static int ublk_queue_init(struct ublk_queue *q) 
	q->cmd_inflight = 0;

	q->tid = gettid();



	if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) {

	if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) {

		q->state |= UBLKSRV_NO_BUF;

		if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)

			q->state |= UBLKSRV_ZC;

		if (dev->dev_info.flags & UBLK_F_AUTO_BUF_REG)

			q->state |= UBLKSRV_AUTO_BUF_REG;

	}



	cmd_buf_size = ublk_queue_cmd_buf_sz(q);
@@ -461,7 +464,7 @@ static int ublk_queue_init(struct ublk_queue *q) 
		goto fail;

	}



	if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) {

	if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) {

		ret = io_uring_register_buffers_sparse(&q->ring, q->q_depth);

		if (ret) {

			ublk_err("ublk dev %d queue %d register spare buffers failed %d",
@@ -525,6 +528,18 @@ static void ublk_dev_unprep(struct ublk_dev *dev) 
	close(dev->fds[0]);

}



static void ublk_set_auto_buf_reg(struct io_uring_sqe *sqe,

				  unsigned short buf_idx,

				  unsigned char flags)

{

	struct ublk_auto_buf_reg buf = {

		.index = buf_idx,

		.flags = flags,

	};



	sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf);

}



int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag)

{

	struct ublksrv_io_cmd *cmd;
@@ -579,6 +594,9 @@ int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag) 
	else

		cmd->addr	= 0;



	if (q->state & UBLKSRV_AUTO_BUF_REG)

		ublk_set_auto_buf_reg(sqe[0], tag, 0);



	user_data = build_user_data(tag, _IOC_NR(cmd_op), 0, 0);

	io_uring_sqe_set_data64(sqe[0], user_data);
@@ -1206,6 +1224,7 @@ static int cmd_dev_get_features(void) 
		[const_ilog2(UBLK_F_USER_COPY)] = "USER_COPY",

		[const_ilog2(UBLK_F_ZONED)] = "ZONED",

		[const_ilog2(UBLK_F_USER_RECOVERY_FAIL_IO)] = "RECOVERY_FAIL_IO",

		[const_ilog2(UBLK_F_AUTO_BUF_REG)] = "AUTO_BUF_REG",

	};

	struct ublk_dev *dev;

	__u64 features = 0;
@@ -1245,7 +1264,7 @@ static void __cmd_create_help(char *exe, bool recovery) 


	printf("%s %s -t [null|loop|stripe|fault_inject] [-q nr_queues] [-d depth] [-n dev_id]\n",

			exe, recovery ? "recover" : "add");

	printf("\t[--foreground] [--quiet] [-z] [--debug_mask mask] [-r 0|1 ] [-g]\n");

	printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--debug_mask mask] [-r 0|1 ] [-g]\n");

	printf("\t[-e 0|1 ] [-i 0|1]\n");

	printf("\t[target options] [backfile1] [backfile2] ...\n");

	printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n");
@@ -1300,6 +1319,7 @@ int main(int argc, char *argv[]) 
		{ "recovery_fail_io",	1,	NULL, 'e'},

		{ "recovery_reissue",	1,	NULL, 'i'},

		{ "get_data",		1,	NULL, 'g'},

		{ "auto_zc",		0,	NULL,  0},

		{ 0, 0, 0, 0 }

	};

	const struct ublk_tgt_ops *ops = NULL;
@@ -1368,6 +1388,8 @@ int main(int argc, char *argv[]) 
				ublk_dbg_mask = 0;

			if (!strcmp(longopts[option_idx].name, "foreground"))

				ctx.fg = 1;

			if (!strcmp(longopts[option_idx].name, "auto_zc"))

				ctx.flags |= UBLK_F_AUTO_BUF_REG;

			break;

		case '?':

			/*

tools/testing/selftests/ublk/kublk.h

+7 −0
Original line number Diff line number Diff line
@@ -115,6 +115,7 @@ struct ublk_io { 
#define UBLKSRV_NEED_COMMIT_RQ_COMP	(1UL << 1)

#define UBLKSRV_IO_FREE			(1UL << 2)

#define UBLKSRV_NEED_GET_DATA           (1UL << 3)

#define UBLKSRV_NEED_REG_BUF            (1UL << 4)

	unsigned short flags;

	unsigned short refs;		/* used by target code only */
@@ -168,6 +169,7 @@ struct ublk_queue { 
#define UBLKSRV_QUEUE_IDLE	(1U << 1)

#define UBLKSRV_NO_BUF		(1U << 2)

#define UBLKSRV_ZC		(1U << 3)

#define UBLKSRV_AUTO_BUF_REG		(1U << 4)

	unsigned state;

	pid_t tid;

	pthread_t thread;
@@ -387,6 +389,11 @@ static inline int ublk_queue_use_zc(const struct ublk_queue *q) 
	return q->state & UBLKSRV_ZC;

}



static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q)

{

	return q->state & UBLKSRV_AUTO_BUF_REG;

}



extern const struct ublk_tgt_ops null_tgt_ops;

extern const struct ublk_tgt_ops loop_tgt_ops;

extern const struct ublk_tgt_ops stripe_tgt_ops;

tools/testing/selftests/ublk/null.c

+32 −11
Original line number Diff line number Diff line
@@ -42,10 +42,22 @@ static int ublk_null_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev) 
	return 0;

}



static void __setup_nop_io(int tag, const struct ublksrv_io_desc *iod,

		struct io_uring_sqe *sqe)

{

	unsigned ublk_op = ublksrv_get_op(iod);



	io_uring_prep_nop(sqe);

	sqe->buf_index = tag;

	sqe->flags |= IOSQE_FIXED_FILE;

	sqe->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;

	sqe->len = iod->nr_sectors << 9; 	/* injected result */

	sqe->user_data = build_user_data(tag, ublk_op, 0, 1);

}



static int null_queue_zc_io(struct ublk_queue *q, int tag)

{

	const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);

	unsigned ublk_op = ublksrv_get_op(iod);

	struct io_uring_sqe *sqe[3];



	ublk_queue_alloc_sqes(q, sqe, 3);
@@ -55,12 +67,8 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag) 
			ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);

	sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;



	io_uring_prep_nop(sqe[1]);

	sqe[1]->buf_index = tag;

	sqe[1]->flags |= IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK;

	sqe[1]->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;

	sqe[1]->len = iod->nr_sectors << 9; 	/* injected result */

	sqe[1]->user_data = build_user_data(tag, ublk_op, 0, 1);

	__setup_nop_io(tag, iod, sqe[1]);

	sqe[1]->flags |= IOSQE_IO_HARDLINK;



	io_uring_prep_buf_unregister(sqe[2], 0, tag, q->q_id, tag);

	sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, 1);
@@ -69,6 +77,16 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag) 
	return 2;

}



static int null_queue_auto_zc_io(struct ublk_queue *q, int tag)

{

	const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);

	struct io_uring_sqe *sqe[1];



	ublk_queue_alloc_sqes(q, sqe, 1);

	__setup_nop_io(tag, iod, sqe[0]);

	return 1;

}



static void ublk_null_io_done(struct ublk_queue *q, int tag,

		const struct io_uring_cqe *cqe)

{
@@ -94,15 +112,18 @@ static void ublk_null_io_done(struct ublk_queue *q, int tag, 
static int ublk_null_queue_io(struct ublk_queue *q, int tag)

{

	const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);

	int zc = ublk_queue_use_zc(q);

	unsigned auto_zc = ublk_queue_use_auto_zc(q);

	unsigned zc = ublk_queue_use_zc(q);

	int queued;



	if (!zc) {

	if (auto_zc)

		queued = null_queue_auto_zc_io(q, tag);

	else if (zc)

		queued = null_queue_zc_io(q, tag);

	else {

		ublk_complete_io(q, tag, iod->nr_sectors << 9);

		return 0;

	}



	queued = null_queue_zc_io(q, tag);

	ublk_queued_tgt_io(q, tag, queued);

	return 0;

}