Commit f0e4b1b6 authored by George Guo's avatar George Guo Committed by Huacai Chen
Browse files

LoongArch: Add 128-bit atomic cmpxchg support 

Implement 128-bit atomic compare-and-exchange using LoongArch's
LL.D/SC.Q instructions.

At the same time, this fix the BPF scheduler test failures (scx_central
and scx_qmap) caused by kmalloc_nolock_noprof() returning NULL, due to
missing 128-bit atomics. The NULL returns lead to -ENOMEM errors during
scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler
(located in tools/sched_ext/).

Building with `make` and running ./tools/sched_ext/build/bin/scx_qmap.

Link: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/commit/?id=5fb750e8a9ae


Acked-by: default avatarHengqi Chen <hengqi.chen@gmail.com>
Tested-by: default avatarHengqi Chen <hengqi.chen@gmail.com>
Co-developed-by: default avatar: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: default avatarXi Ruoyao <xry111@xry111.site>
Signed-off-by: default avatarGeorge Guo <guodongtai@kylinos.cn>
Signed-off-by: default avatarHuacai Chen <chenhuacai@loongson.cn>
parent 48543c42

arch/loongarch/Kconfig

+2 −0
Original line number Diff line number Diff line
@@ -114,6 +114,7 @@ config LOONGARCH 
	select GENERIC_TIME_VSYSCALL

	select GPIOLIB

	select HAS_IOPORT

	select HAVE_ALIGNED_STRUCT_PAGE

	select HAVE_ARCH_AUDITSYSCALL

	select HAVE_ARCH_BITREVERSE

	select HAVE_ARCH_JUMP_LABEL
@@ -130,6 +131,7 @@ config LOONGARCH 
	select HAVE_ARCH_TRANSPARENT_HUGEPAGE

	select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD

	select HAVE_ASM_MODVERSIONS

	select HAVE_CMPXCHG_DOUBLE

	select HAVE_CMPXCHG_LOCAL

	select HAVE_CONTEXT_TRACKING_USER

	select HAVE_C_RECORDMCOUNT

arch/loongarch/include/asm/cmpxchg.h

+54 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
#include <linux/bits.h>

#include <linux/build_bug.h>

#include <asm/barrier.h>

#include <asm/cpu-features.h>



#define __xchg_amo_asm(amswap_db, m, val)	\

({						\
@@ -236,6 +237,59 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int 
	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\

	arch_cmpxchg((ptr), (o), (n));					\

  })



union __u128_halves {

	u128 full;

	struct {

		u64 low;

		u64 high;

	};

};



#define system_has_cmpxchg128()	cpu_opt(LOONGARCH_CPU_SCQ)



#define __arch_cmpxchg128(ptr, old, new, llsc_mb)			\

({									\

	union __u128_halves __old, __new, __ret;			\

	volatile u64 *__ptr = (volatile u64 *)(ptr);			\

									\

	__old.full = (old);                                             \

	__new.full = (new);						\

									\

	__asm__ __volatile__(						\

	"1:   ll.d    %0, %3		# 128-bit cmpxchg low	\n"	\

	llsc_mb								\

	"     ld.d    %1, %4		# 128-bit cmpxchg high	\n"	\

	"     move    $t0, %0					\n"	\

	"     move    $t1, %1					\n"	\

	"     bne     %0, %z5, 2f				\n"	\

	"     bne     %1, %z6, 2f				\n"	\

	"     move    $t0, %z7					\n"	\

	"     move    $t1, %z8					\n"	\

	"2:   sc.q    $t0, $t1, %2				\n"	\

	"     beqz    $t0, 1b					\n"	\

	llsc_mb								\

	: "=&r" (__ret.low), "=&r" (__ret.high)				\

	: "r" (__ptr),							\

	  "ZC" (__ptr[0]), "m" (__ptr[1]),				\

	  "Jr" (__old.low), "Jr" (__old.high),				\

	  "Jr" (__new.low), "Jr" (__new.high)				\

	: "t0", "t1", "memory");					\

									\

	__ret.full;							\

})



#define arch_cmpxchg128(ptr, o, n)					\

({									\

	BUILD_BUG_ON(sizeof(*(ptr)) != 16);				\

	__arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB);			\

})



#define arch_cmpxchg128_local(ptr, o, n)				\

({									\

	BUILD_BUG_ON(sizeof(*(ptr)) != 16);				\

	__arch_cmpxchg128(ptr, o, n, "");				\

})

#else

#include <asm-generic/cmpxchg-local.h>

#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))