Unverified Commit 1658ef43 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Implement cmpxchg8/16() using Zabha 



This adds runtime support for Zabha in cmpxchg8/16() operations.

Note that in the absence of Zacas support in the toolchain, CAS
instructions from Zabha won't be used.

Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: default avatarAndrew Jones <ajones@ventanamicro.com>
Reviewed-by: default avatarAndrea Parri <parri.andrea@gmail.com>
Link: https://lore.kernel.org/r/20241103145153.105097-6-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 51624ddc

arch/riscv/Kconfig

+18 −0
Original line number Diff line number Diff line
@@ -632,6 +632,24 @@ config RISCV_ISA_ZAWRS 
	  use of these instructions in the kernel when the Zawrs extension is

	  detected at boot.



config TOOLCHAIN_HAS_ZABHA

	bool

	default y

	depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zabha)

	depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zabha)

	depends on AS_HAS_OPTION_ARCH



config RISCV_ISA_ZABHA

	bool "Zabha extension support for atomic byte/halfword operations"

	depends on TOOLCHAIN_HAS_ZABHA

	depends on RISCV_ALTERNATIVE

	default y

	help

	  Enable the use of the Zabha ISA-extension to implement kernel

	  byte/halfword atomic memory operations when it is detected at boot.



	  If you don't know what to do here, say Y.



config TOOLCHAIN_HAS_ZACAS

	bool

	default y

arch/riscv/Makefile

+3 −0
Original line number Diff line number Diff line
@@ -85,6 +85,9 @@ endif 
# Check if the toolchain supports Zacas

riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas



# Check if the toolchain supports Zabha

riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha



# Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by

# matching non-v and non-multi-letter extensions out with the filter ([^v_]*)

KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')

arch/riscv/include/asm/cmpxchg.h

+49 −29
Original line number Diff line number Diff line
@@ -108,8 +108,22 @@ 
 * indicated by comparing RETURN with OLD.

 */



#define __arch_cmpxchg_masked(sc_sfx, prepend, append, r, p, o, n)	\

#define __arch_cmpxchg_masked(sc_sfx, cas_sfx, prepend, append, r, p, o, n)	\

({										\

	if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) &&				\

	    IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&				\

	    riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) &&		\

	    riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) {		\

		r = o;								\

										\

		__asm__ __volatile__ (						\

			prepend							\

			"	amocas" cas_sfx " %0, %z2, %1\n"		\

			append							\

			: "+&r" (r), "+A" (*(p))				\

			: "rJ" (n)						\

			: "memory");						\

	} else {								\

		u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3);			\

		ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE;	\

		ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0)	\
@@ -136,6 +150,7 @@ 
			: "memory");						\

										\

		r = (__typeof__(*(p)))((__retx & __mask) >> __s);		\

	}									\

})



#define __arch_cmpxchg(lr_sfx, sc_cas_sfx, prepend, append, r, p, co, o, n)	\
@@ -177,8 +192,13 @@ 
									\

	switch (sizeof(*__ptr)) {					\

	case 1:								\

		__arch_cmpxchg_masked(sc_cas_sfx, ".b" sc_cas_sfx,	\

					prepend, append,		\

					__ret, __ptr, __old, __new);    \

		break;							\

	case 2:								\

		__arch_cmpxchg_masked(sc_cas_sfx, prepend, append,	\

		__arch_cmpxchg_masked(sc_cas_sfx, ".h" sc_cas_sfx,	\

					prepend, append,		\

					__ret, __ptr, __old, __new);	\

		break;							\

	case 4:								\

arch/riscv/include/asm/hwcap.h

+1 −0
Original line number Diff line number Diff line
@@ -93,6 +93,7 @@ 
#define RISCV_ISA_EXT_ZCMOP		84

#define RISCV_ISA_EXT_ZAWRS		85

#define RISCV_ISA_EXT_SVVPTC		86

#define RISCV_ISA_EXT_ZABHA		87



#define RISCV_ISA_EXT_XLINUXENVCFG	127

arch/riscv/kernel/cpufeature.c

+1 −0
Original line number Diff line number Diff line
@@ -322,6 +322,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { 
	__RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),

	__RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),

	__RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),

	__RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA),

	__RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),

	__RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),

	__RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA),