Unverified Commit f7bd2be7 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Implement arch_cmpxchg128() using Zacas 



Now that Zacas is supported in the kernel, let's use the double word
atomic version of amocas to improve the SLUB allocator.

Note that we have to select fixed registers, otherwise gcc fails to pick
even registers and then produces a reserved encoding which fails to
assemble.

Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: default avatarAndrew Jones <ajones@ventanamicro.com>
Reviewed-by: default avatarAndrea Parri <parri.andrea@gmail.com>
Link: https://lore.kernel.org/r/20241103145153.105097-8-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 6116e22e

arch/riscv/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -115,6 +115,7 @@ config RISCV 
	select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO

	select HARDIRQS_SW_RESEND

	select HAS_IOPORT if MMU

	select HAVE_ALIGNED_STRUCT_PAGE

	select HAVE_ARCH_AUDITSYSCALL

	select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP

	select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT

arch/riscv/include/asm/cmpxchg.h

+38 −0
Original line number Diff line number Diff line
@@ -296,6 +296,44 @@ 
	arch_cmpxchg_release((ptr), (o), (n));				\

})



#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS)



#define system_has_cmpxchg128()        riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)



union __u128_halves {

	u128 full;

	struct {

		u64 low, high;

	};

};



#define __arch_cmpxchg128(p, o, n, cas_sfx)					\

({										\

	__typeof__(*(p)) __o = (o);                                             \

	union __u128_halves __hn = { .full = (n) };				\

	union __u128_halves __ho = { .full = (__o) };				\

	register unsigned long t1 asm ("t1") = __hn.low;			\

	register unsigned long t2 asm ("t2") = __hn.high;			\

	register unsigned long t3 asm ("t3") = __ho.low;			\

	register unsigned long t4 asm ("t4") = __ho.high;			\

										\

	__asm__ __volatile__ (							\

		 "       amocas.q" cas_sfx " %0, %z3, %2"			\

		 : "+&r" (t3), "+&r" (t4), "+A" (*(p))				\

		 : "rJ" (t1), "rJ" (t2)						\

		 : "memory");							\

										\

		 ((u128)t4 << 64) | t3;						\

})



#define arch_cmpxchg128(ptr, o, n)						\

	__arch_cmpxchg128((ptr), (o), (n), ".aqrl")



#define arch_cmpxchg128_local(ptr, o, n)					\

	__arch_cmpxchg128((ptr), (o), (n), "")



#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS */



#ifdef CONFIG_RISCV_ISA_ZAWRS

/*

 * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to