Commit 32034df6 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Ingo Molnar
Browse files

rseq: Switch to TIF_RSEQ if supported 



TIF_NOTIFY_RESUME is a multiplexing TIF bit, which is suboptimal especially
with the RSEQ fast path depending on it, but not really handling it.

Define a separate TIF_RSEQ in the generic TIF space and enable the full
separation of fast and slow path for architectures which utilize that.

That avoids the hassle with invocations of resume_user_mode_work() from
hypervisors, which clear TIF_NOTIFY_RESUME. It makes the therefore required
re-evaluation at the end of vcpu_run() a NOOP on architectures which
utilize the generic TIF space and have a separate TIF_RSEQ.

The hypervisor TIF handling does not include the separate TIF_RSEQ as there
is no point in doing so. The guest does neither know nor care about the VMM
host applications RSEQ state. That state is only relevant when the ioctl()
returns to user space.

The fastpath implementation still utilizes TIF_NOTIFY_RESUME for failure
handling, but this only happens within exit_to_user_mode_loop(), so
arguably the hypervisor ioctl() code is long done when this happens.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarMathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251027084307.903622031@linutronix.de
parent 7a5201ea

include/asm-generic/thread_info_tif.h

+3 −0
Original line number Diff line number Diff line
@@ -45,4 +45,7 @@ 
# define _TIF_RESTORE_SIGMASK	BIT(TIF_RESTORE_SIGMASK)

#endif



#define TIF_RSEQ		11	// Run RSEQ fast path

#define _TIF_RSEQ		BIT(TIF_RSEQ)



#endif /* _ASM_GENERIC_THREAD_INFO_TIF_H_ */

include/linux/irq-entry-common.h

+1 −1
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ 
#define EXIT_TO_USER_MODE_WORK						\

	(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |		\

	 _TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY |			\

	 _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL |			\

	 _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | _TIF_RSEQ |		\

	 ARCH_EXIT_TO_USER_MODE_WORK)



/**

include/linux/rseq.h

+15 −7
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *reg 


static inline void rseq_raise_notify_resume(struct task_struct *t)

{

	set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);

	set_tsk_thread_flag(t, TIF_RSEQ);

}



/* Invoked from context switch to force evaluation on exit to user */
@@ -114,17 +114,25 @@ static inline void rseq_force_update(void) 


/*

 * KVM/HYPERV invoke resume_user_mode_work() before entering guest mode,

 * which clears TIF_NOTIFY_RESUME. To avoid updating user space RSEQ in

 * that case just to do it eventually again before returning to user space,

 * the entry resume_user_mode_work() invocation is ignored as the register

 * argument is NULL.

 * which clears TIF_NOTIFY_RESUME on architectures that don't use the

 * generic TIF bits and therefore can't provide a separate TIF_RSEQ flag.

 *

 * After returning from guest mode, they have to invoke this function to

 * re-raise TIF_NOTIFY_RESUME if necessary.

 * To avoid updating user space RSEQ in that case just to do it eventually

 * again before returning to user space, because __rseq_handle_slowpath()

 * does nothing when invoked with NULL register state.

 *

 * After returning from guest mode, before exiting to userspace, hypervisors

 * must invoke this function to re-raise TIF_NOTIFY_RESUME if necessary.

 */

static inline void rseq_virt_userspace_exit(void)

{

	if (current->rseq.event.sched_switch)

	/*

	 * The generic optimization for deferring RSEQ updates until the next

	 * exit relies on having a dedicated TIF_RSEQ.

	 */

	if (!IS_ENABLED(CONFIG_HAVE_GENERIC_TIF_BITS) &&

	    current->rseq.event.sched_switch)

		rseq_raise_notify_resume(current);

}

include/linux/rseq_entry.h

+29 −3
Original line number Diff line number Diff line
@@ -507,18 +507,44 @@ static __always_inline bool __rseq_exit_to_user_mode_restart(struct pt_regs *reg 
	return false;

}



static __always_inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs)

/* Required to allow conversion to GENERIC_ENTRY w/o GENERIC_TIF_BITS */

#ifdef CONFIG_HAVE_GENERIC_TIF_BITS

static __always_inline bool test_tif_rseq(unsigned long ti_work)

{

	return ti_work & _TIF_RSEQ;

}



static __always_inline void clear_tif_rseq(void)

{

	static_assert(TIF_RSEQ != TIF_NOTIFY_RESUME);

	clear_thread_flag(TIF_RSEQ);

}

#else

static __always_inline bool test_tif_rseq(unsigned long ti_work) { return true; }

static __always_inline void clear_tif_rseq(void) { }

#endif



static __always_inline bool

rseq_exit_to_user_mode_restart(struct pt_regs *regs, unsigned long ti_work)

{

	if (likely(!test_tif_rseq(ti_work)))

		return false;



	if (unlikely(__rseq_exit_to_user_mode_restart(regs))) {

		current->rseq.event.slowpath = true;

		set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);

		return true;

	}



	clear_tif_rseq();

	return false;

}



#else /* CONFIG_GENERIC_ENTRY */

static inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs) { return false; }

static inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs, unsigned long ti_work)

{

	return false;

}

#endif /* !CONFIG_GENERIC_ENTRY */



static __always_inline void rseq_syscall_exit_to_user_mode(void)
@@ -577,7 +603,7 @@ static inline void rseq_debug_syscall_return(struct pt_regs *regs) 
}

#else /* CONFIG_RSEQ */

static inline void rseq_note_user_irq_entry(void) { }

static inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs)

static inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs, unsigned long ti_work)

{

	return false;

}

include/linux/thread_info.h

+5 −0
Original line number Diff line number Diff line
@@ -67,6 +67,11 @@ enum syscall_work_bit { 
#define _TIF_NEED_RESCHED_LAZY _TIF_NEED_RESCHED

#endif



#ifndef TIF_RSEQ

# define TIF_RSEQ	TIF_NOTIFY_RESUME

# define _TIF_RSEQ	_TIF_NOTIFY_RESUME

#endif



#ifdef __KERNEL__



#ifndef arch_set_restart_data