Unverified Commit 07a0a41c authored by Leonardo Bras's avatar Leonardo Bras Committed by Palmer Dabbelt
Browse files

riscv/cmpxchg: Deduplicate cmpxchg() asm and macros 



In this header every cmpxchg define (_relaxed, _acquire, _release,
vanilla) contain it's own asm file, both for 4-byte variables an 8-byte
variables, on a total of 8 versions of mostly the same asm.

This is usually bad, as it means any change may be done in up to 8
different places.

Unify those versions by creating a new define with enough parameters to
generate any version of the previous 8.

Then unify the result under a more general define, and simplify
arch_cmpxchg* generation

(This did not cause any change in generated asm)

Signed-off-by: default avatarLeonardo Bras <leobras@redhat.com>
Reviewed-by: default avatarGuo Ren <guoren@kernel.org>
Reviewed-by: default avatarAndrea Parri <parri.andrea@gmail.com>
Tested-by: default avatarGuo Ren <guoren@kernel.org>
Link: https://lore.kernel.org/r/20240103163203.72768-4-leobras@redhat.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 4bfa185f

arch/riscv/include/asm/cmpxchg.h

+33 −162
Original line number Diff line number Diff line
@@ -71,190 +71,61 @@ 
 * store NEW in MEM.  Return the initial value in MEM.  Success is

 * indicated by comparing RETURN with OLD.

 */

#define __cmpxchg_relaxed(ptr, old, new, size)				\

({									\

	__typeof__(ptr) __ptr = (ptr);					\

	__typeof__(*(ptr)) __old = (old);				\

	__typeof__(*(ptr)) __new = (new);				\

	__typeof__(*(ptr)) __ret;					\

	register unsigned int __rc;					\

	switch (size) {							\

	case 4:								\

		__asm__ __volatile__ (					\

			"0:	lr.w %0, %2\n"				\

			"	bne  %0, %z3, 1f\n"			\

			"	sc.w %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" ((long)__old), "rJ" (__new)		\

			: "memory");					\

		break;							\

	case 8:								\

		__asm__ __volatile__ (					\

			"0:	lr.d %0, %2\n"				\

			"	bne %0, %z3, 1f\n"			\

			"	sc.d %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" (__old), "rJ" (__new)			\

			: "memory");					\

		break;							\

	default:							\

		BUILD_BUG();						\

	}								\

	__ret;								\

})



#define arch_cmpxchg_relaxed(ptr, o, n)					\

({									\

	__typeof__(*(ptr)) _o_ = (o);					\

	__typeof__(*(ptr)) _n_ = (n);					\

	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),			\

					_o_, _n_, sizeof(*(ptr)));	\

})



#define __cmpxchg_acquire(ptr, old, new, size)				\

#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n)	\

({									\

	__typeof__(ptr) __ptr = (ptr);					\

	__typeof__(*(ptr)) __old = (old);				\

	__typeof__(*(ptr)) __new = (new);				\

	__typeof__(*(ptr)) __ret;					\

	register unsigned int __rc;					\

	switch (size) {							\

	case 4:								\

		__asm__ __volatile__ (					\

			"0:	lr.w %0, %2\n"				\

			"	bne  %0, %z3, 1f\n"			\

			"	sc.w %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			RISCV_ACQUIRE_BARRIER				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" ((long)__old), "rJ" (__new)		\

			: "memory");					\

		break;							\

	case 8:								\

									\

	__asm__ __volatile__ (						\

			"0:	lr.d %0, %2\n"				\

		prepend							\

		"0:	lr" lr_sfx " %0, %2\n"				\

		"	bne  %0, %z3, 1f\n"				\

			"	sc.d %1, %z4, %2\n"			\

		"	sc" sc_sfx " %1, %z4, %2\n"			\

		"	bnez %1, 0b\n"					\

			RISCV_ACQUIRE_BARRIER				\

		append							\

		"1:\n"							\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" (__old), "rJ" (__new)			\

		: "=&r" (r), "=&r" (__rc), "+A" (*(p))			\

		: "rJ" (co o), "rJ" (n)					\

		: "memory");						\

		break;							\

	default:							\

		BUILD_BUG();						\

	}								\

	__ret;								\

})



#define arch_cmpxchg_acquire(ptr, o, n)					\

({									\

	__typeof__(*(ptr)) _o_ = (o);					\

	__typeof__(*(ptr)) _n_ = (n);					\

	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr),			\

					_o_, _n_, sizeof(*(ptr)));	\

})



#define __cmpxchg_release(ptr, old, new, size)				\

#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append)		\

({									\

	__typeof__(ptr) __ptr = (ptr);					\

	__typeof__(*(ptr)) __old = (old);				\

	__typeof__(*(ptr)) __new = (new);				\

	__typeof__(*(ptr)) __ret;					\

	register unsigned int __rc;					\

	switch (size) {							\

	__typeof__(*(__ptr)) __old = (old);				\

	__typeof__(*(__ptr)) __new = (new);				\

	__typeof__(*(__ptr)) __ret;					\

									\

	switch (sizeof(*__ptr)) {					\

	case 4:								\

		__asm__ __volatile__ (					\

			RISCV_RELEASE_BARRIER				\

			"0:	lr.w %0, %2\n"				\

			"	bne  %0, %z3, 1f\n"			\

			"	sc.w %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" ((long)__old), "rJ" (__new)		\

			: "memory");					\

		__arch_cmpxchg(".w", ".w" sc_sfx, prepend, append,	\

				__ret, __ptr, (long), __old, __new);	\

		break;							\

	case 8:								\

		__asm__ __volatile__ (					\

			RISCV_RELEASE_BARRIER				\

			"0:	lr.d %0, %2\n"				\

			"	bne %0, %z3, 1f\n"			\

			"	sc.d %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" (__old), "rJ" (__new)			\

			: "memory");					\

		__arch_cmpxchg(".d", ".d" sc_sfx, prepend, append,	\

				__ret, __ptr, /**/, __old, __new);	\

		break;							\

	default:							\

		BUILD_BUG();						\

	}								\

	__ret;								\

	(__typeof__(*(__ptr)))__ret;					\

})



#define arch_cmpxchg_release(ptr, o, n)					\

({									\

	__typeof__(*(ptr)) _o_ = (o);					\

	__typeof__(*(ptr)) _n_ = (n);					\

	(__typeof__(*(ptr))) __cmpxchg_release((ptr),			\

					_o_, _n_, sizeof(*(ptr)));	\

})

#define arch_cmpxchg_relaxed(ptr, o, n)					\

	_arch_cmpxchg((ptr), (o), (n), "", "", "")



#define __cmpxchg(ptr, old, new, size)					\

({									\

	__typeof__(ptr) __ptr = (ptr);					\

	__typeof__(*(ptr)) __old = (old);				\

	__typeof__(*(ptr)) __new = (new);				\

	__typeof__(*(ptr)) __ret;					\

	register unsigned int __rc;					\

	switch (size) {							\

	case 4:								\

		__asm__ __volatile__ (					\

			"0:	lr.w %0, %2\n"				\

			"	bne  %0, %z3, 1f\n"			\

			"	sc.w.rl %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			"	fence rw, rw\n"				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" ((long)__old), "rJ" (__new)		\

			: "memory");					\

		break;							\

	case 8:								\

		__asm__ __volatile__ (					\

			"0:	lr.d %0, %2\n"				\

			"	bne %0, %z3, 1f\n"			\

			"	sc.d.rl %1, %z4, %2\n"			\

			"	bnez %1, 0b\n"				\

			"	fence rw, rw\n"				\

			"1:\n"						\

			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\

			: "rJ" (__old), "rJ" (__new)			\

			: "memory");					\

		break;							\

	default:							\

		BUILD_BUG();						\

	}								\

	__ret;								\

})

#define arch_cmpxchg_acquire(ptr, o, n)					\

	_arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)



#define arch_cmpxchg_release(ptr, o, n)					\

	_arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")



#define arch_cmpxchg(ptr, o, n)						\

({									\

	__typeof__(*(ptr)) _o_ = (o);					\

	__typeof__(*(ptr)) _n_ = (n);					\

	(__typeof__(*(ptr))) __cmpxchg((ptr),				\

				       _o_, _n_, sizeof(*(ptr)));	\

})

	_arch_cmpxchg((ptr), (o), (n), ".rl", "", "	fence rw, rw\n")



#define arch_cmpxchg_local(ptr, o, n)					\

	(__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))

	arch_cmpxchg_relaxed((ptr), (o), (n))



#define arch_cmpxchg64(ptr, o, n)					\

({									\