Commit 90611bb3 authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge branch 'for-6.15/io_uring' into for-6.15/io_uring-epoll-wait 

* for-6.15/io_uring: (30 commits)
  io_uring: use lockless_cq flag in io_req_complete_post()
  io_uring: pass struct io_tw_state by value
  io_uring: introduce type alias for io_tw_state
  io_uring/rsrc: avoid NULL check in io_put_rsrc_node()
  io_uring: pass ctx instead of req to io_init_req_drain()
  io_uring: use IO_REQ_LINK_FLAGS more
  io_uring/net: improve recv bundles
  io_uring/waitid: use generic io_cancel_remove() helper
  io_uring/futex: use generic io_cancel_remove() helper
  io_uring/cancel: add generic cancel helper
  io_uring/waitid: convert to io_cancel_remove_all()
  io_uring/futex: convert to io_cancel_remove_all()
  io_uring/cancel: add generic remove_all helper
  io_uring/kbuf: uninline __io_put_kbufs
  io_uring/kbuf: introduce io_kbuf_drop_legacy()
  io_uring/kbuf: open code __io_put_kbuf()
  io_uring/kbuf: remove legacy kbuf caching
  io_uring/kbuf: simplify __io_put_kbuf
  io_uring/kbuf: move locking into io_kbuf_drop()
  io_uring/kbuf: remove legacy kbuf kmem cache
  ...
parents 87a132e7 62aa9805

include/linux/io_uring_types.h

+8 −4
Original line number Diff line number Diff line
@@ -360,7 +360,6 @@ struct io_ring_ctx { 


	spinlock_t		completion_lock;



	struct list_head	io_buffers_comp;

	struct list_head	cq_overflow_list;



	struct hlist_head	waitid_list;
@@ -379,8 +378,6 @@ struct io_ring_ctx { 
	unsigned int		file_alloc_start;

	unsigned int		file_alloc_end;



	struct list_head	io_buffers_cache;



	/* Keep this last, we don't need it for the fast path */

	struct wait_queue_head		poll_wq;

	struct io_restriction		restrictions;
@@ -439,8 +436,15 @@ struct io_ring_ctx { 
	struct io_mapped_region		param_region;

};



/*

 * Token indicating function is called in task work context:

 * ctx->uring_lock is held and any completions generated will be flushed.

 * ONLY core io_uring.c should instantiate this struct.

 */

struct io_tw_state {

};

/* Alias to use in code that doesn't instantiate struct io_tw_state */

typedef struct io_tw_state io_tw_token_t;



enum {

	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
@@ -566,7 +570,7 @@ enum { 
	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),

};



typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);

typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);



struct io_task_work {

	struct llist_node		node;

io_uring/cancel.c

+42 −0
Original line number Diff line number Diff line
@@ -341,3 +341,45 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) 
		fput(file);

	return ret;

}



bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,

			  struct hlist_head *list, bool cancel_all,

			  bool (*cancel)(struct io_kiocb *))

{

	struct hlist_node *tmp;

	struct io_kiocb *req;

	bool found = false;



	lockdep_assert_held(&ctx->uring_lock);



	hlist_for_each_entry_safe(req, tmp, list, hash_node) {

		if (!io_match_task_safe(req, tctx, cancel_all))

			continue;

		hlist_del_init(&req->hash_node);

		if (cancel(req))

			found = true;

	}



	return found;

}



int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,

		     unsigned int issue_flags, struct hlist_head *list,

		     bool (*cancel)(struct io_kiocb *))

{

	struct hlist_node *tmp;

	struct io_kiocb *req;

	int nr = 0;



	io_ring_submit_lock(ctx, issue_flags);

	hlist_for_each_entry_safe(req, tmp, list, hash_node) {

		if (!io_cancel_req_match(req, cd))

			continue;

		if (cancel(req))

			nr++;

		if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))

			break;

	}

	io_ring_submit_unlock(ctx, issue_flags);

	return nr ?: -ENOENT;

}

io_uring/cancel.h

+8 −0
Original line number Diff line number Diff line
@@ -24,6 +24,14 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, 
int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg);

bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd);



bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,

			  struct hlist_head *list, bool cancel_all,

			  bool (*cancel)(struct io_kiocb *));



int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,

		     unsigned int issue_flags, struct hlist_head *list,

		     bool (*cancel)(struct io_kiocb *));



static inline bool io_cancel_match_sequence(struct io_kiocb *req, int sequence)

{

	if (req->cancel_seq_set && sequence == req->work.cancel_seq)

io_uring/futex.c

+11 −47
Original line number Diff line number Diff line
@@ -44,30 +44,30 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) 
	io_alloc_cache_free(&ctx->futex_cache, kfree);

}



static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	req->async_data = NULL;

	hlist_del_init(&req->hash_node);

	io_req_task_complete(req, ts);

	io_req_task_complete(req, tw);

}



static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	struct io_futex_data *ifd = req->async_data;

	struct io_ring_ctx *ctx = req->ctx;



	io_tw_lock(ctx, ts);

	io_tw_lock(ctx, tw);

	if (!io_alloc_cache_put(&ctx->futex_cache, ifd))

		kfree(ifd);

	__io_futex_complete(req, ts);

	__io_futex_complete(req, tw);

}



static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)

static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)

{

	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);

	struct futex_vector *futexv = req->async_data;



	io_tw_lock(req->ctx, ts);

	io_tw_lock(req->ctx, tw);



	if (!iof->futexv_unqueued) {

		int res;
@@ -79,7 +79,7 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts) 


	kfree(req->async_data);

	req->flags &= ~REQ_F_ASYNC_DATA;

	__io_futex_complete(req, ts);

	__io_futex_complete(req, tw);

}



static bool io_futexv_claim(struct io_futex *iof)
@@ -90,7 +90,7 @@ static bool io_futexv_claim(struct io_futex *iof) 
	return true;

}



static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)

static bool __io_futex_cancel(struct io_kiocb *req)

{

	/* futex wake already done or in progress */

	if (req->opcode == IORING_OP_FUTEX_WAIT) {
@@ -116,49 +116,13 @@ static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) 
int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,

		    unsigned int issue_flags)

{

	struct hlist_node *tmp;

	struct io_kiocb *req;

	int nr = 0;



	if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_FD_FIXED))

		return -ENOENT;



	io_ring_submit_lock(ctx, issue_flags);

	hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {

		if (req->cqe.user_data != cd->data &&

		    !(cd->flags & IORING_ASYNC_CANCEL_ANY))

			continue;

		if (__io_futex_cancel(ctx, req))

			nr++;

		if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))

			break;

	}

	io_ring_submit_unlock(ctx, issue_flags);



	if (nr)

		return nr;



	return -ENOENT;

	return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel);

}



bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,

			 bool cancel_all)

{

	struct hlist_node *tmp;

	struct io_kiocb *req;

	bool found = false;



	lockdep_assert_held(&ctx->uring_lock);



	hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {

		if (!io_match_task_safe(req, tctx, cancel_all))

			continue;

		hlist_del_init(&req->hash_node);

		__io_futex_cancel(ctx, req);

		found = true;

	}



	return found;

	return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel);

}



int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)

io_uring/io-wq.c

+131 −99

File changed.

Preview size limit exceeded, changes collapsed.