Commit 44adf2bf authored by Yeoreum Yun's avatar Yeoreum Yun Committed by Catalin Marinas
Browse files

arm64: futex: Support futex with FEAT_LSUI 



Current futex atomic operations are implemented using LL/SC instructions
while temporarily clearing PSTATE.PAN and setting PSTATE.TCO (if
KASAN_HW_TAGS is enabled). With Armv9.6, FEAT_LSUI provides atomic
instructions for user memory access in the kernel without the need for
PSTATE bits toggling.

Use the FEAT_LSUI instructions to implement the futex atomic operations.
Note that some futex operations do not have a matching LSUI instruction,
(eor or word-sized cmpxchg). For such cases, use cas{al}t to implement
the operation.

Signed-off-by: default avatarYeoreum Yun <yeoreum.yun@arm.com>
[catalin.marinas@arm.com: add comment on -EAGAIN in __lsui_futex_cmpxchg()]
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent eaa3babc

arch/arm64/include/asm/futex.h

+158 −2
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
#include <linux/uaccess.h>



#include <asm/errno.h>

#include <asm/lsui.h>



#define FUTEX_MAX_LOOPS	128 /* What's the largest number you can think of? */
@@ -89,11 +90,166 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) 
	return ret;

}



#ifdef CONFIG_ARM64_LSUI



/*

 * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as

 * PAN toggling is not required.

 */



#define LSUI_FUTEX_ATOMIC_OP(op, asm_op)				\

static __always_inline int						\

__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval)	\

{									\

	int ret = 0;							\

	int oldval;							\

									\

	uaccess_ttbr0_enable();						\

									\

	asm volatile("// __lsui_futex_atomic_" #op "\n"			\

	__LSUI_PREAMBLE							\

"1:	" #asm_op "al	%w[oparg], %w[oldval], %[uaddr]\n"		\

"2:\n"									\

	_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])			\

	: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),			\

	  [oldval] "=r" (oldval)					\

	: [oparg] "r" (oparg)						\

	: "memory");							\

									\

	uaccess_ttbr0_disable();					\

									\

	if (!ret)							\

		*oval = oldval;						\

	return ret;							\

}



LSUI_FUTEX_ATOMIC_OP(add, ldtadd)

LSUI_FUTEX_ATOMIC_OP(or, ldtset)

LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr)

LSUI_FUTEX_ATOMIC_OP(set, swpt)



static __always_inline int

__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval)

{

	int ret = 0;



	uaccess_ttbr0_enable();



	asm volatile("// __lsui_cmpxchg64\n"

	__LSUI_PREAMBLE

"1:	casalt	%[oldval], %[newval], %[uaddr]\n"

"2:\n"

	_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])

	: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),

	  [oldval] "+r" (*oldval)

	: [newval] "r" (newval)

	: "memory");



	uaccess_ttbr0_disable();



	return ret;

}



static __always_inline int

__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)

{

	u64 __user *uaddr64;

	bool futex_pos, other_pos;

	u32 other, orig_other;

	union {

		u32 futex[2];

		u64 raw;

	} oval64, orig64, nval64;



	uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64));

	futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64));

	other_pos = !futex_pos;



	oval64.futex[futex_pos] = oldval;

	if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))

		return -EFAULT;



	orig64.raw = oval64.raw;



	nval64.futex[futex_pos] = newval;

	nval64.futex[other_pos] = oval64.futex[other_pos];



	if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw))

		return -EFAULT;



	oldval = oval64.futex[futex_pos];

	other = oval64.futex[other_pos];

	orig_other = orig64.futex[other_pos];



	if (other != orig_other)

		return -EAGAIN;



	*oval = oldval;



	return 0;

}



static __always_inline int

__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval)

{

	/*

	 * Undo the bitwise negation applied to the oparg passed from

	 * arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN.

	 */

	return __lsui_futex_atomic_andnot(~oparg, uaddr, oval);

}



static __always_inline int

__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)

{

	u32 oldval, newval, val;

	int ret, i;



	if (get_user(oldval, uaddr))

		return -EFAULT;



	/*

	 * there are no ldteor/stteor instructions...

	 */

	for (i = 0; i < FUTEX_MAX_LOOPS; i++) {

		newval = oldval ^ oparg;



		ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val);

		switch (ret) {

		case -EFAULT:

			return ret;

		case -EAGAIN:

			continue;

		}



		if (val == oldval) {

			*oval = val;

			return 0;

		}



		oldval = val;

	}



	return -EAGAIN;

}



static __always_inline int

__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)

{

	/*

	 * Callers of futex_atomic_cmpxchg_inatomic() already retry on

	 * -EAGAIN, no need for another loop of max retries.

	 */

	return __lsui_cmpxchg32(uaddr, oldval, newval, oval);

}

#endif	/* CONFIG_ARM64_LSUI */





#define FUTEX_ATOMIC_OP(op)						\

static __always_inline int						\

__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval)		\

{									\

	return __llsc_futex_atomic_##op(oparg, uaddr, oval);		\

	return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval);	\

}



FUTEX_ATOMIC_OP(add)
@@ -105,7 +261,7 @@ FUTEX_ATOMIC_OP(set) 
static __always_inline int

__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)

{

	return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval);

	return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval);

}



static inline int

arch/arm64/include/asm/lsui.h

0 → 100644
+27 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef __ASM_LSUI_H

#define __ASM_LSUI_H



#include <linux/compiler_types.h>

#include <linux/stringify.h>

#include <asm/alternative.h>

#include <asm/alternative-macros.h>

#include <asm/cpucaps.h>



#define __LSUI_PREAMBLE	".arch_extension lsui\n"



#ifdef CONFIG_ARM64_LSUI



#define __lsui_llsc_body(op, ...)					\

({									\

	alternative_has_cap_unlikely(ARM64_HAS_LSUI) ?			\

		__lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__);	\

})



#else	/* CONFIG_ARM64_LSUI */



#define __lsui_llsc_body(op, ...)	__llsc_##op(__VA_ARGS__)



#endif	/* CONFIG_ARM64_LSUI */



#endif	/* __ASM_LSUI_H */