Commit eaa3babc authored by Yeoreum Yun's avatar Yeoreum Yun Committed by Catalin Marinas
Browse files

arm64: futex: Refactor futex atomic operation 



Refactor the futex atomic operations using ll/sc instructions in
preparation for FEAT_LSUI support. In addition, use named operands for
the inline asm.

No functional change.

Signed-off-by: default avatarYeoreum Yun <yeoreum.yun@arm.com>
[catalin.marinas@arm.com: remove unnecessary stringify.h include]
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 42550d7d

arch/arm64/include/asm/futex.h

+97 −58
Original line number Diff line number Diff line
@@ -12,68 +12,133 @@ 


#define FUTEX_MAX_LOOPS	128 /* What's the largest number you can think of? */



#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\

do {									\

#define LLSC_FUTEX_ATOMIC_OP(op, insn)					\

static __always_inline int						\

__llsc_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval)	\

{									\

	unsigned int loops = FUTEX_MAX_LOOPS;				\

	int ret, oldval, newval;					\

									\

	uaccess_enable_privileged();					\

	asm volatile(							\

"	prfm	pstl1strm, %2\n"					\

"1:	ldxr	%w1, %2\n"						\

	asm volatile("// __llsc_futex_atomic_" #op "\n"			\

"	prfm	pstl1strm, %[uaddr]\n"					\

"1:	ldxr	%w[oldval], %[uaddr]\n"					\

	insn "\n"							\

"2:	stlxr	%w0, %w3, %2\n"						\

"	cbz	%w0, 3f\n"						\

"	sub	%w4, %w4, %w0\n"					\

"	cbnz	%w4, 1b\n"						\

"	mov	%w0, %w6\n"						\

"2:	stlxr	%w[ret], %w[newval], %[uaddr]\n"			\

"	cbz	%w[ret], 3f\n"						\

"	sub	%w[loops], %w[loops], %w[ret]\n"			\

"	cbnz	%w[loops], 1b\n"					\

"	mov	%w[ret], %w[err]\n"					\

"3:\n"									\

"	dmb	ish\n"							\

	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0)				\

	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0)				\

	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),	\

	  "+r" (loops)							\

	: "r" (oparg), "Ir" (-EAGAIN)					\

	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret])			\

	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret])			\

	: [ret] "=&r" (ret), [oldval] "=&r" (oldval),			\

	  [uaddr] "+Q" (*uaddr), [newval] "=&r" (newval),		\

	  [loops] "+r" (loops)						\

	: [oparg] "r" (oparg), [err] "Ir" (-EAGAIN)			\

	: "memory");							\

	uaccess_disable_privileged();					\

} while (0)

									\

	if (!ret)							\

		*oval = oldval;						\

									\

	return ret;							\

}



LLSC_FUTEX_ATOMIC_OP(add, "add	%w[newval], %w[oldval], %w[oparg]")

LLSC_FUTEX_ATOMIC_OP(or,  "orr	%w[newval], %w[oldval], %w[oparg]")

LLSC_FUTEX_ATOMIC_OP(and, "and	%w[newval], %w[oldval], %w[oparg]")

LLSC_FUTEX_ATOMIC_OP(eor, "eor	%w[newval], %w[oldval], %w[oparg]")

LLSC_FUTEX_ATOMIC_OP(set, "mov	%w[newval], %w[oparg]")



static __always_inline int

__llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)

{

	int ret = 0;

	unsigned int loops = FUTEX_MAX_LOOPS;

	u32 val, tmp;



	uaccess_enable_privileged();

	asm volatile("//__llsc_futex_cmpxchg\n"

"	prfm	pstl1strm, %[uaddr]\n"

"1:	ldxr	%w[curval], %[uaddr]\n"

"	eor	%w[tmp], %w[curval], %w[oldval]\n"

"	cbnz	%w[tmp], 4f\n"

"2:	stlxr	%w[tmp], %w[newval], %[uaddr]\n"

"	cbz	%w[tmp], 3f\n"

"	sub	%w[loops], %w[loops], %w[tmp]\n"

"	cbnz	%w[loops], 1b\n"

"	mov	%w[ret], %w[err]\n"

"3:\n"

"	dmb	ish\n"

"4:\n"

	_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w[ret])

	_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w[ret])

	: [ret] "+r" (ret), [curval] "=&r" (val),

	  [uaddr] "+Q" (*uaddr), [tmp] "=&r" (tmp),

	  [loops] "+r" (loops)

	: [oldval] "r" (oldval), [newval] "r" (newval),

	  [err] "Ir" (-EAGAIN)

	: "memory");

	uaccess_disable_privileged();



	if (!ret)

		*oval = val;



	return ret;

}



#define FUTEX_ATOMIC_OP(op)						\

static __always_inline int						\

__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval)		\

{									\

	return __llsc_futex_atomic_##op(oparg, uaddr, oval);		\

}



FUTEX_ATOMIC_OP(add)

FUTEX_ATOMIC_OP(or)

FUTEX_ATOMIC_OP(and)

FUTEX_ATOMIC_OP(eor)

FUTEX_ATOMIC_OP(set)



static __always_inline int

__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)

{

	return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval);

}



static inline int

arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)

{

	int oldval = 0, ret, tmp;

	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);

	int ret;

	u32 __user *uaddr;



	if (!access_ok(_uaddr, sizeof(u32)))

		return -EFAULT;



	uaddr = __uaccess_mask_ptr(_uaddr);



	switch (op) {

	case FUTEX_OP_SET:

		__futex_atomic_op("mov	%w3, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		ret = __futex_atomic_set(oparg, uaddr, oval);

		break;

	case FUTEX_OP_ADD:

		__futex_atomic_op("add	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		ret = __futex_atomic_add(oparg, uaddr, oval);

		break;

	case FUTEX_OP_OR:

		__futex_atomic_op("orr	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		ret = __futex_atomic_or(oparg, uaddr, oval);

		break;

	case FUTEX_OP_ANDN:

		__futex_atomic_op("and	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, ~oparg);

		ret = __futex_atomic_and(~oparg, uaddr, oval);

		break;

	case FUTEX_OP_XOR:

		__futex_atomic_op("eor	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		ret = __futex_atomic_eor(oparg, uaddr, oval);

		break;

	default:

		ret = -ENOSYS;

	}



	if (!ret)

		*oval = oldval;



	return ret;

}
@@ -81,40 +146,14 @@ static inline int 
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,

			      u32 oldval, u32 newval)

{

	int ret = 0;

	unsigned int loops = FUTEX_MAX_LOOPS;

	u32 val, tmp;

	u32 __user *uaddr;



	if (!access_ok(_uaddr, sizeof(u32)))

		return -EFAULT;



	uaddr = __uaccess_mask_ptr(_uaddr);

	uaccess_enable_privileged();

	asm volatile("// futex_atomic_cmpxchg_inatomic\n"

"	prfm	pstl1strm, %2\n"

"1:	ldxr	%w1, %2\n"

"	sub	%w3, %w1, %w5\n"

"	cbnz	%w3, 4f\n"

"2:	stlxr	%w3, %w6, %2\n"

"	cbz	%w3, 3f\n"

"	sub	%w4, %w4, %w3\n"

"	cbnz	%w4, 1b\n"

"	mov	%w0, %w7\n"

"3:\n"

"	dmb	ish\n"

"4:\n"

	_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)

	_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)

	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)

	: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)

	: "memory");

	uaccess_disable_privileged();



	if (!ret)

		*uval = val;



	return ret;

	return __futex_cmpxchg(uaddr, oldval, newval, uval);

}



#endif /* __ASM_FUTEX_H */