Commit 3597f278 authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring/rsrc: unify file and buffer resource tables 



For files, there's nr_user_files/file_table/file_data, and buffers have
nr_user_bufs/user_bufs/buf_data. There's no reason why file_table and
file_data can't be the same thing, and ditto for the buffer side. That
gets rid of more io_ring_ctx state that's in two spots rather than just
being in one spot, as it should be. Put all the registered file data in
one locations, and ditto on the buffer front.

This also avoids having both io_rsrc_data->nodes being an allocated
array, and ->user_bufs[] or ->file_table.nodes. There's no reason to
have this information duplicated. Keep it in one spot, io_rsrc_data,
along with how many resources are available.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f38f2847

include/linux/io_uring_types.h

+7 −8
Original line number Diff line number Diff line
@@ -55,8 +55,13 @@ struct io_wq_work { 
	int cancel_seq;

};



struct io_file_table {

struct io_rsrc_data {

	unsigned int			nr;

	struct io_rsrc_node		**nodes;

};



struct io_file_table {

	struct io_rsrc_data data;

	unsigned long *bitmap;

	unsigned int alloc_hint;

};
@@ -276,9 +281,7 @@ struct io_ring_ctx { 
		struct io_wq_work_list	iopoll_list;



		struct io_file_table	file_table;

		struct io_rsrc_node	**user_bufs;

		unsigned		nr_user_files;

		unsigned		nr_user_bufs;

		struct io_rsrc_data	buf_table;



		struct io_submit_state	submit_state;
@@ -366,10 +369,6 @@ struct io_ring_ctx { 
	struct wait_queue_head		poll_wq;

	struct io_restriction		restrictions;



	/* slow path rsrc auxilary data, used by update/register */

	struct io_rsrc_data		*file_data;

	struct io_rsrc_data		*buf_data;



	u32			pers_next;

	struct xarray		personalities;

io_uring/cancel.c

+2 −2
Original line number Diff line number Diff line
@@ -240,9 +240,9 @@ static int __io_sync_cancel(struct io_uring_task *tctx, 
	/* fixed must be grabbed every time since we drop the uring_lock */

	if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&

	    (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {

		if (unlikely(fd >= ctx->nr_user_files))

		if (unlikely(fd >= ctx->file_table.data.nr))

			return -EBADF;

		fd = array_index_nospec(fd, ctx->nr_user_files);

		fd = array_index_nospec(fd, ctx->file_table.data.nr);

		cd->file = io_file_from_index(&ctx->file_table, fd);

		if (!cd->file)

			return -EBADF;

io_uring/fdinfo.c

+5 −5
Original line number Diff line number Diff line
@@ -165,8 +165,8 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) 
	seq_printf(m, "SqThreadCpu:\t%d\n", sq_cpu);

	seq_printf(m, "SqTotalTime:\t%llu\n", sq_total_time);

	seq_printf(m, "SqWorkTime:\t%llu\n", sq_work_time);

	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);

	for (i = 0; has_lock && i < ctx->nr_user_files; i++) {

	seq_printf(m, "UserFiles:\t%u\n", ctx->file_table.data.nr);

	for (i = 0; has_lock && i < ctx->file_table.data.nr; i++) {

		struct file *f = io_file_from_index(&ctx->file_table, i);



		if (f)
@@ -174,9 +174,9 @@ __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *file) 
		else

			seq_printf(m, "%5u: <none>\n", i);

	}

	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);

	for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {

		struct io_mapped_ubuf *buf = ctx->user_bufs[i]->buf;

	seq_printf(m, "UserBufs:\t%u\n", ctx->buf_table.nr);

	for (i = 0; has_lock && i < ctx->buf_table.nr; i++) {

		struct io_mapped_ubuf *buf = ctx->buf_table.nodes[i]->buf;



		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len);

	}

io_uring/filetable.c

+19 −25
Original line number Diff line number Diff line
@@ -38,25 +38,19 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) 


bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)

{

	table->nodes = kvmalloc_array(nr_files, sizeof(struct io_src_node *),

					GFP_KERNEL_ACCOUNT | __GFP_ZERO);

	if (unlikely(!table->nodes))

	if (io_rsrc_data_alloc(&table->data, nr_files))

		return false;



	table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);

	if (unlikely(!table->bitmap)) {

		kvfree(table->nodes);

		return false;

	}



	if (table->bitmap)

		return true;

	io_rsrc_data_free(&table->data);

	return false;

}



void io_free_file_tables(struct io_file_table *table)

{

	kvfree(table->nodes);

	io_rsrc_data_free(&table->data);

	bitmap_free(table->bitmap);

	table->nodes = NULL;

	table->bitmap = NULL;

}
@@ -68,22 +62,22 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file, 


	if (io_is_uring_fops(file))

		return -EBADF;

	if (!ctx->file_data)

	if (!ctx->file_table.data.nr)

		return -ENXIO;

	if (slot_index >= ctx->nr_user_files)

	if (slot_index >= ctx->file_table.data.nr)

		return -EINVAL;



	node = io_rsrc_node_alloc(ctx, IORING_RSRC_FILE);

	if (!node)

		return -ENOMEM;



	slot_index = array_index_nospec(slot_index, ctx->nr_user_files);

	if (ctx->file_table.nodes[slot_index])

		io_put_rsrc_node(ctx->file_table.nodes[slot_index]);

	slot_index = array_index_nospec(slot_index, ctx->file_table.data.nr);

	if (ctx->file_table.data.nodes[slot_index])

		io_put_rsrc_node(ctx->file_table.data.nodes[slot_index]);

	else

		io_file_bitmap_set(&ctx->file_table, slot_index);



	ctx->file_table.nodes[slot_index] = node;

	ctx->file_table.data.nodes[slot_index] = node;

	io_fixed_file_set(node, file);

	return 0;

}
@@ -129,16 +123,16 @@ int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags, 


int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)

{

	if (unlikely(!ctx->file_data))

	if (unlikely(!ctx->file_table.data.nr))

		return -ENXIO;

	if (offset >= ctx->nr_user_files)

	if (offset >= ctx->file_table.data.nr)

		return -EINVAL;



	offset = array_index_nospec(offset, ctx->nr_user_files);

	if (!ctx->file_table.nodes[offset])

	offset = array_index_nospec(offset, ctx->file_table.data.nr);

	if (!ctx->file_table.data.nodes[offset])

		return -EBADF;

	io_put_rsrc_node(ctx->file_table.nodes[offset]);

	ctx->file_table.nodes[offset] = NULL;

	io_put_rsrc_node(ctx->file_table.data.nodes[offset]);

	ctx->file_table.data.nodes[offset] = NULL;

	io_file_bitmap_clear(&ctx->file_table, offset);

	return 0;

}
@@ -153,7 +147,7 @@ int io_register_file_alloc_range(struct io_ring_ctx *ctx, 
		return -EFAULT;

	if (check_add_overflow(range.off, range.len, &end))

		return -EOVERFLOW;

	if (range.resv || end > ctx->nr_user_files)

	if (range.resv || end > ctx->file_table.data.nr)

		return -EINVAL;



	io_file_table_set_alloc_range(ctx, range.off, range.len);

io_uring/filetable.h

+1 −1
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ static inline struct file *io_slot_file(struct io_rsrc_node *node) 
static inline struct file *io_file_from_index(struct io_file_table *table,

					      int index)

{

	struct io_rsrc_node *node = table->nodes[index];

	struct io_rsrc_node *node = table->data.nodes[index];



	if (node)

		return io_slot_file(node);