Unverified Commit 38acdee3 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Implement cmpxchg32/64() using Zacas 



This adds runtime support for Zacas in cmpxchg operations.

Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: default avatarAndrew Jones <ajones@ventanamicro.com>
Reviewed-by: default avatarAndrea Parri <parri.andrea@gmail.com>
Link: https://lore.kernel.org/r/20241103145153.105097-4-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent af042c45

arch/riscv/Kconfig

+16 −0
Original line number Diff line number Diff line
@@ -632,6 +632,22 @@ config RISCV_ISA_ZAWRS 
	  use of these instructions in the kernel when the Zawrs extension is

	  detected at boot.



config TOOLCHAIN_HAS_ZACAS

	bool

	default y

	depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zacas)

	depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zacas)

	depends on AS_HAS_OPTION_ARCH



config RISCV_ISA_ZACAS

	bool "Zacas extension support for atomic CAS"

	depends on TOOLCHAIN_HAS_ZACAS

	depends on RISCV_ALTERNATIVE

	default y

	help

	  Enable the use of the Zacas ISA-extension to implement kernel atomic

	  cmpxchg operations when it is detected at boot.



	  If you don't know what to do here, say Y.



config TOOLCHAIN_HAS_ZBB

arch/riscv/Makefile

+3 −0
Original line number Diff line number Diff line
@@ -82,6 +82,9 @@ else 
riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei

endif



# Check if the toolchain supports Zacas

riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas



# Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by

# matching non-v and non-multi-letter extensions out with the filter ([^v_]*)

KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')

arch/riscv/include/asm/cmpxchg.h

+31 −17
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
#include <asm/fence.h>

#include <asm/hwcap.h>

#include <asm/insn-def.h>

#include <asm/cpufeature-macros.h>



#define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n)		\

({									\
@@ -137,24 +138,37 @@ 
	r = (__typeof__(*(p)))((__retx & __mask) >> __s);		\

})



#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n)	\

#define __arch_cmpxchg(lr_sfx, sc_cas_sfx, prepend, append, r, p, co, o, n)	\

({									\

	if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&			\

	    riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) {	\

		r = o;							\

									\

		__asm__ __volatile__ (					\

			prepend						\

			"	amocas" sc_cas_sfx " %0, %z2, %1\n"	\

			append						\

			: "+&r" (r), "+A" (*(p))			\

			: "rJ" (n)					\

			: "memory");					\

	} else {							\

		register unsigned int __rc;				\

									\

		__asm__ __volatile__ (					\

			prepend						\

			"0:	lr" lr_sfx " %0, %2\n"			\

			"	bne  %0, %z3, 1f\n"			\

		"	sc" sc_sfx " %1, %z4, %2\n"			\

			"	sc" sc_cas_sfx " %1, %z4, %2\n"		\

			"	bnez %1, 0b\n"				\

			append						\

			"1:\n"						\

			: "=&r" (r), "=&r" (__rc), "+A" (*(p))		\

			: "rJ" (co o), "rJ" (n)				\

			: "memory");					\

	}								\

})



#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append)		\

#define _arch_cmpxchg(ptr, old, new, sc_cas_sfx, prepend, append)	\

({									\

	__typeof__(ptr) __ptr = (ptr);					\

	__typeof__(*(__ptr)) __old = (old);				\
@@ -164,15 +178,15 @@ 
	switch (sizeof(*__ptr)) {					\

	case 1:								\

	case 2:								\

		__arch_cmpxchg_masked(sc_sfx, prepend, append,		\

		__arch_cmpxchg_masked(sc_cas_sfx, prepend, append,	\

					__ret, __ptr, __old, __new);	\

		break;							\

	case 4:								\

		__arch_cmpxchg(".w", ".w" sc_sfx, prepend, append,	\

		__arch_cmpxchg(".w", ".w" sc_cas_sfx, prepend, append,	\

				__ret, __ptr, (long), __old, __new);	\

		break;							\

	case 8:								\

		__arch_cmpxchg(".d", ".d" sc_sfx, prepend, append,	\

		__arch_cmpxchg(".d", ".d" sc_cas_sfx, prepend, append,	\

				__ret, __ptr, /**/, __old, __new);	\

		break;							\

	default:							\