Commit 194bb58c authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: add support for futex wake and wait 



Add support for FUTEX_WAKE/WAIT primitives.

IORING_OP_FUTEX_WAKE is mix of FUTEX_WAKE and FUTEX_WAKE_BITSET, as
it does support passing in a bitset.

Similary, IORING_OP_FUTEX_WAIT is a mix of FUTEX_WAIT and
FUTEX_WAIT_BITSET.

For both of them, they are using the futex2 interface.

FUTEX_WAKE is straight forward, as those can always be done directly from
the io_uring submission without needing async handling. For FUTEX_WAIT,
things are a bit more complicated. If the futex isn't ready, then we
rely on a callback via futex_queue->wake() when someone wakes up the
futex. From that calback, we queue up task_work with the original task,
which will post a CQE and wake it, if necessary.

Cancelations are supported, both from the application point-of-view,
but also to be able to cancel pending waits if the ring exits before
all events have occurred. The return value of futex_unqueue() is used
to gate who wins the potential race between cancelation and futex
wakeups. Whomever gets a 'ret == 1' return from that claims ownership
of the io_uring futex request.

This is just the barebones wait/wake support. PI or REQUEUE support is
not added at this point, unclear if we might look into that later.

Likewise, explicit timeouts are not supported either. It is expected
that users that need timeouts would do so via the usual io_uring
mechanism to do that using linked timeouts.

The SQE format is as follows:

`addr`		Address of futex
`fd`		futex2(2) FUTEX2_* flags
`futex_flags`	io_uring specific command flags. None valid now.
`addr2`		Value of futex
`addr3`		Mask to wake/wait

Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e52c4340

include/linux/io_uring_types.h

+5 −0
Original line number Diff line number Diff line
@@ -321,6 +321,11 @@ struct io_ring_ctx { 


	struct hlist_head	waitid_list;



#ifdef CONFIG_FUTEX

	struct hlist_head	futex_list;

	struct io_alloc_cache	futex_cache;

#endif



	const struct cred	*sq_creds;	/* cred used for __io_sq_thread() */

	struct io_sq_data	*sq_data;	/* if using sq thread polling */

include/uapi/linux/io_uring.h

+3 −0
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ struct io_uring_sqe { 
		__u32		msg_ring_flags;

		__u32		uring_cmd_flags;

		__u32		waitid_flags;

		__u32		futex_flags;

	};

	__u64	user_data;	/* data to be passed back at completion time */

	/* pack this to avoid bogus arm OABI complaints */
@@ -243,6 +244,8 @@ enum io_uring_op { 
	IORING_OP_SENDMSG_ZC,

	IORING_OP_READ_MULTISHOT,

	IORING_OP_WAITID,

	IORING_OP_FUTEX_WAIT,

	IORING_OP_FUTEX_WAKE,



	/* this goes last, obviously */

	IORING_OP_LAST,

io_uring/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -10,3 +10,4 @@ obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ 
					cancel.o kbuf.o rsrc.o rw.o opdef.o \

					notif.o waitid.o

obj-$(CONFIG_IO_WQ)		+= io-wq.o

obj-$(CONFIG_FUTEX)		+= futex.o

io_uring/cancel.c

+5 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 
#include "poll.h"

#include "timeout.h"

#include "waitid.h"

#include "futex.h"

#include "cancel.h"



struct io_cancel {
@@ -124,6 +125,10 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, 
	if (ret != -ENOENT)

		return ret;



	ret = io_futex_cancel(ctx, cd, issue_flags);

	if (ret != -ENOENT)

		return ret;



	spin_lock(&ctx->completion_lock);

	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))

		ret = io_timeout_cancel(ctx, cd);

io_uring/cancel.h

+4 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

#ifndef IORING_CANCEL_H

#define IORING_CANCEL_H



#include <linux/io_uring_types.h>
@@ -22,3 +24,5 @@ void init_hash_table(struct io_hash_table *table, unsigned size); 


int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg);

bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd);



#endif