Commit 79974cc3 authored by Huacai Chen's avatar Huacai Chen
Browse files

LoongArch: Add atomic operations for 32BIT/64BIT 



LoongArch64 has both AMO and LL/SC instructions, while LoongArch32 only
has LL/SC intstructions. So we add a Kconfig option CPU_HAS_AMO here and
implement atomic operations (also including local operations and percpu
operations) for both 32BIT and 64BIT platforms.

Reviewed-by: default avatarArnd Bergmann <arnd@arndb.de>
Signed-off-by: default avatarJiaxun Yang <jiaxun.yang@flygoat.com>
Signed-off-by: default avatarHuacai Chen <chenhuacai@loongson.cn>
parent bf3fa8f2

arch/loongarch/Kconfig

+4 −0
Original line number Diff line number Diff line
@@ -568,6 +568,10 @@ config ARCH_STRICT_ALIGN 
	  to run kernel only on systems with h/w unaligned access support in

	  order to optimise for performance.



config CPU_HAS_AMO

	bool

	default 64BIT



config CPU_HAS_FPU

	bool

	default y

arch/loongarch/include/asm/atomic-amo.h

0 → 100644
+206 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * Atomic operations (AMO).

 *

 * Copyright (C) 2020-2025 Loongson Technology Corporation Limited

 */



#ifndef _ASM_ATOMIC_AMO_H

#define _ASM_ATOMIC_AMO_H



#include <linux/types.h>

#include <asm/barrier.h>

#include <asm/cmpxchg.h>



#define ATOMIC_OP(op, I, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	__asm__ __volatile__(						\

	"am"#asm_op".w" " $zero, %1, %0	\n"				\

	: "+ZB" (v->counter)						\

	: "r" (I)							\

	: "memory");							\

}



#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix)		\

static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v)	\

{									\

	int result;							\

									\

	__asm__ __volatile__(						\

	"am"#asm_op#mb".w" " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)				\

	: "r" (I)							\

	: "memory");							\

									\

	return result c_op I;						\

}



#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix)			\

static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v)	\

{									\

	int result;							\

									\

	__asm__ __volatile__(						\

	"am"#asm_op#mb".w" " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)				\

	: "r" (I)							\

	: "memory");							\

									\

	return result;							\

}



#define ATOMIC_OPS(op, I, asm_op, c_op)					\

	ATOMIC_OP(op, I, asm_op)					\

	ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db,         )		\

	ATOMIC_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)		\

	ATOMIC_FETCH_OP(op, I, asm_op, _db,         )			\

	ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC_OPS(add, i, add, +)

ATOMIC_OPS(sub, -i, add, +)



#define arch_atomic_add_return		arch_atomic_add_return

#define arch_atomic_add_return_acquire	arch_atomic_add_return

#define arch_atomic_add_return_release	arch_atomic_add_return

#define arch_atomic_add_return_relaxed	arch_atomic_add_return_relaxed

#define arch_atomic_sub_return		arch_atomic_sub_return

#define arch_atomic_sub_return_acquire	arch_atomic_sub_return

#define arch_atomic_sub_return_release	arch_atomic_sub_return

#define arch_atomic_sub_return_relaxed	arch_atomic_sub_return_relaxed

#define arch_atomic_fetch_add		arch_atomic_fetch_add

#define arch_atomic_fetch_add_acquire	arch_atomic_fetch_add

#define arch_atomic_fetch_add_release	arch_atomic_fetch_add

#define arch_atomic_fetch_add_relaxed	arch_atomic_fetch_add_relaxed

#define arch_atomic_fetch_sub		arch_atomic_fetch_sub

#define arch_atomic_fetch_sub_acquire	arch_atomic_fetch_sub

#define arch_atomic_fetch_sub_release	arch_atomic_fetch_sub

#define arch_atomic_fetch_sub_relaxed	arch_atomic_fetch_sub_relaxed



#undef ATOMIC_OPS



#define ATOMIC_OPS(op, I, asm_op)					\

	ATOMIC_OP(op, I, asm_op)					\

	ATOMIC_FETCH_OP(op, I, asm_op, _db,         )			\

	ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC_OPS(and, i, and)

ATOMIC_OPS(or, i, or)

ATOMIC_OPS(xor, i, xor)



#define arch_atomic_fetch_and		arch_atomic_fetch_and

#define arch_atomic_fetch_and_acquire	arch_atomic_fetch_and

#define arch_atomic_fetch_and_release	arch_atomic_fetch_and

#define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed

#define arch_atomic_fetch_or		arch_atomic_fetch_or

#define arch_atomic_fetch_or_acquire	arch_atomic_fetch_or

#define arch_atomic_fetch_or_release	arch_atomic_fetch_or

#define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed

#define arch_atomic_fetch_xor		arch_atomic_fetch_xor

#define arch_atomic_fetch_xor_acquire	arch_atomic_fetch_xor

#define arch_atomic_fetch_xor_release	arch_atomic_fetch_xor

#define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#ifdef CONFIG_64BIT



#define ATOMIC64_OP(op, I, asm_op)					\

static inline void arch_atomic64_##op(long i, atomic64_t *v)		\

{									\

	__asm__ __volatile__(						\

	"am"#asm_op".d " " $zero, %1, %0	\n"			\

	: "+ZB" (v->counter)						\

	: "r" (I)							\

	: "memory");							\

}



#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix)			\

static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v)	\

{										\

	long result;								\

	__asm__ __volatile__(							\

	"am"#asm_op#mb".d " " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)					\

	: "r" (I)								\

	: "memory");								\

										\

	return result c_op I;							\

}



#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix)				\

static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v)	\

{										\

	long result;								\

										\

	__asm__ __volatile__(							\

	"am"#asm_op#mb".d " " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)					\

	: "r" (I)								\

	: "memory");								\

										\

	return result;								\

}



#define ATOMIC64_OPS(op, I, asm_op, c_op)				      \

	ATOMIC64_OP(op, I, asm_op)					      \

	ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db,         )		      \

	ATOMIC64_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)		      \

	ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )			      \

	ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC64_OPS(add, i, add, +)

ATOMIC64_OPS(sub, -i, add, +)



#define arch_atomic64_add_return		arch_atomic64_add_return

#define arch_atomic64_add_return_acquire	arch_atomic64_add_return

#define arch_atomic64_add_return_release	arch_atomic64_add_return

#define arch_atomic64_add_return_relaxed	arch_atomic64_add_return_relaxed

#define arch_atomic64_sub_return		arch_atomic64_sub_return

#define arch_atomic64_sub_return_acquire	arch_atomic64_sub_return

#define arch_atomic64_sub_return_release	arch_atomic64_sub_return

#define arch_atomic64_sub_return_relaxed	arch_atomic64_sub_return_relaxed

#define arch_atomic64_fetch_add			arch_atomic64_fetch_add

#define arch_atomic64_fetch_add_acquire		arch_atomic64_fetch_add

#define arch_atomic64_fetch_add_release		arch_atomic64_fetch_add

#define arch_atomic64_fetch_add_relaxed		arch_atomic64_fetch_add_relaxed

#define arch_atomic64_fetch_sub			arch_atomic64_fetch_sub

#define arch_atomic64_fetch_sub_acquire		arch_atomic64_fetch_sub

#define arch_atomic64_fetch_sub_release		arch_atomic64_fetch_sub

#define arch_atomic64_fetch_sub_relaxed		arch_atomic64_fetch_sub_relaxed



#undef ATOMIC64_OPS



#define ATOMIC64_OPS(op, I, asm_op)					      \

	ATOMIC64_OP(op, I, asm_op)					      \

	ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )			      \

	ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC64_OPS(and, i, and)

ATOMIC64_OPS(or, i, or)

ATOMIC64_OPS(xor, i, xor)



#define arch_atomic64_fetch_and		arch_atomic64_fetch_and

#define arch_atomic64_fetch_and_acquire	arch_atomic64_fetch_and

#define arch_atomic64_fetch_and_release	arch_atomic64_fetch_and

#define arch_atomic64_fetch_and_relaxed	arch_atomic64_fetch_and_relaxed

#define arch_atomic64_fetch_or		arch_atomic64_fetch_or

#define arch_atomic64_fetch_or_acquire	arch_atomic64_fetch_or

#define arch_atomic64_fetch_or_release	arch_atomic64_fetch_or

#define arch_atomic64_fetch_or_relaxed	arch_atomic64_fetch_or_relaxed

#define arch_atomic64_fetch_xor		arch_atomic64_fetch_xor

#define arch_atomic64_fetch_xor_acquire	arch_atomic64_fetch_xor

#define arch_atomic64_fetch_xor_release	arch_atomic64_fetch_xor

#define arch_atomic64_fetch_xor_relaxed	arch_atomic64_fetch_xor_relaxed



#undef ATOMIC64_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



#endif



#endif /* _ASM_ATOMIC_AMO_H */

arch/loongarch/include/asm/atomic-llsc.h

0 → 100644
+100 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * Atomic operations (LLSC).

 *

 * Copyright (C) 2024-2025 Loongson Technology Corporation Limited

 */



#ifndef _ASM_ATOMIC_LLSC_H

#define _ASM_ATOMIC_LLSC_H



#include <linux/types.h>

#include <asm/barrier.h>

#include <asm/cmpxchg.h>



#define ATOMIC_OP(op, I, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	int temp;							\

									\

	__asm__ __volatile__(						\

	"1:	ll.w		%0, %1      #atomic_" #op "	\n"	\

	"       " #asm_op "	%0, %0, %2			\n"	\

	"	sc.w		%0, %1				\n"	\

	"       beq		%0, $r0, 1b			\n"	\

	:"=&r" (temp) , "+ZC"(v->counter)				\

	:"r" (I)							\

	);								\

}



#define ATOMIC_OP_RETURN(op, I, asm_op)					\

static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v)	\

{									\

	int result, temp;						\

									\

	__asm__ __volatile__(						\

	"1:     ll.w		%1, %2      # atomic_" #op "_return \n"	\

	"       " #asm_op "	%0, %1, %3                          \n"	\

	"       sc.w		%0, %2                              \n"	\

	"       beq		%0, $r0 ,1b                         \n"	\

	"       " #asm_op "	%0, %1, %3                          \n"	\

	: "=&r" (result), "=&r" (temp),	"+ZC"(v->counter)		\

	: "r" (I));							\

									\

	return result;							\

}



#define ATOMIC_FETCH_OP(op, I, asm_op)					\

static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\

{									\

	int result, temp;						\

									\

	__asm__ __volatile__(						\

	"1:     ll.w		%1, %2      # atomic_fetch_" #op "  \n"	\

	"       " #asm_op "	%0, %1, %3                          \n" \

	"       sc.w		%0, %2                              \n"	\

	"       beq		%0, $r0 ,1b                         \n"	\

	"       add.w		%0, %1  ,$r0                        \n"	\

	: "=&r" (result), "=&r" (temp), "+ZC" (v->counter)		\

	: "r" (I));							\

									\

	return result;							\

}



#define ATOMIC_OPS(op,I ,asm_op, c_op)					\

	ATOMIC_OP(op, I, asm_op)					\

	ATOMIC_OP_RETURN(op, I , asm_op)				\

	ATOMIC_FETCH_OP(op, I, asm_op)



ATOMIC_OPS(add, i , add.w ,+=)

ATOMIC_OPS(sub, -i , add.w ,+=)



#define arch_atomic_add_return_relaxed	arch_atomic_add_return_relaxed

#define arch_atomic_sub_return_relaxed	arch_atomic_sub_return_relaxed

#define arch_atomic_fetch_add_relaxed	arch_atomic_fetch_add_relaxed

#define arch_atomic_fetch_sub_relaxed	arch_atomic_fetch_sub_relaxed



#undef ATOMIC_OPS



#define ATOMIC_OPS(op, I, asm_op)					\

	ATOMIC_OP(op, I, asm_op)					\

	ATOMIC_FETCH_OP(op, I, asm_op)



ATOMIC_OPS(and, i, and)

ATOMIC_OPS(or, i, or)

ATOMIC_OPS(xor, i, xor)



#define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed

#define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed

#define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#ifdef CONFIG_64BIT

#error "64-bit LLSC atomic operations are not supported"

#endif



#endif /* _ASM_ATOMIC_LLSC_H */

arch/loongarch/include/asm/atomic.h

+10 −187
Original line number Diff line number Diff line
@@ -11,6 +11,16 @@ 
#include <asm/barrier.h>

#include <asm/cmpxchg.h>



#ifdef CONFIG_CPU_HAS_AMO

#include <asm/atomic-amo.h>

#else

#include <asm/atomic-llsc.h>

#endif



#ifdef CONFIG_GENERIC_ATOMIC64

#include <asm-generic/atomic64.h>

#endif



#if __SIZEOF_LONG__ == 4

#define __LL		"ll.w	"

#define __SC		"sc.w	"
@@ -34,100 +44,6 @@ 
#define arch_atomic_read(v)	READ_ONCE((v)->counter)

#define arch_atomic_set(v, i)	WRITE_ONCE((v)->counter, (i))



#define ATOMIC_OP(op, I, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	__asm__ __volatile__(						\

	"am"#asm_op".w" " $zero, %1, %0	\n"				\

	: "+ZB" (v->counter)						\

	: "r" (I)							\

	: "memory");							\

}



#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix)		\

static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v)	\

{									\

	int result;							\

									\

	__asm__ __volatile__(						\

	"am"#asm_op#mb".w" " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)				\

	: "r" (I)							\

	: "memory");							\

									\

	return result c_op I;						\

}



#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix)			\

static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v)	\

{									\

	int result;							\

									\

	__asm__ __volatile__(						\

	"am"#asm_op#mb".w" " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)				\

	: "r" (I)							\

	: "memory");							\

									\

	return result;							\

}



#define ATOMIC_OPS(op, I, asm_op, c_op)					\

	ATOMIC_OP(op, I, asm_op)					\

	ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db,         )		\

	ATOMIC_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)		\

	ATOMIC_FETCH_OP(op, I, asm_op, _db,         )			\

	ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC_OPS(add, i, add, +)

ATOMIC_OPS(sub, -i, add, +)



#define arch_atomic_add_return		arch_atomic_add_return

#define arch_atomic_add_return_acquire	arch_atomic_add_return

#define arch_atomic_add_return_release	arch_atomic_add_return

#define arch_atomic_add_return_relaxed	arch_atomic_add_return_relaxed

#define arch_atomic_sub_return		arch_atomic_sub_return

#define arch_atomic_sub_return_acquire	arch_atomic_sub_return

#define arch_atomic_sub_return_release	arch_atomic_sub_return

#define arch_atomic_sub_return_relaxed	arch_atomic_sub_return_relaxed

#define arch_atomic_fetch_add		arch_atomic_fetch_add

#define arch_atomic_fetch_add_acquire	arch_atomic_fetch_add

#define arch_atomic_fetch_add_release	arch_atomic_fetch_add

#define arch_atomic_fetch_add_relaxed	arch_atomic_fetch_add_relaxed

#define arch_atomic_fetch_sub		arch_atomic_fetch_sub

#define arch_atomic_fetch_sub_acquire	arch_atomic_fetch_sub

#define arch_atomic_fetch_sub_release	arch_atomic_fetch_sub

#define arch_atomic_fetch_sub_relaxed	arch_atomic_fetch_sub_relaxed



#undef ATOMIC_OPS



#define ATOMIC_OPS(op, I, asm_op)					\

	ATOMIC_OP(op, I, asm_op)					\

	ATOMIC_FETCH_OP(op, I, asm_op, _db,         )			\

	ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC_OPS(and, i, and)

ATOMIC_OPS(or, i, or)

ATOMIC_OPS(xor, i, xor)



#define arch_atomic_fetch_and		arch_atomic_fetch_and

#define arch_atomic_fetch_and_acquire	arch_atomic_fetch_and

#define arch_atomic_fetch_and_release	arch_atomic_fetch_and

#define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed

#define arch_atomic_fetch_or		arch_atomic_fetch_or

#define arch_atomic_fetch_or_acquire	arch_atomic_fetch_or

#define arch_atomic_fetch_or_release	arch_atomic_fetch_or

#define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed

#define arch_atomic_fetch_xor		arch_atomic_fetch_xor

#define arch_atomic_fetch_xor_acquire	arch_atomic_fetch_xor

#define arch_atomic_fetch_xor_release	arch_atomic_fetch_xor

#define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)

{

       int prev, rc;
@@ -194,99 +110,6 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v) 
#define arch_atomic64_read(v)	READ_ONCE((v)->counter)

#define arch_atomic64_set(v, i)	WRITE_ONCE((v)->counter, (i))



#define ATOMIC64_OP(op, I, asm_op)					\

static inline void arch_atomic64_##op(long i, atomic64_t *v)		\

{									\

	__asm__ __volatile__(						\

	"am"#asm_op".d " " $zero, %1, %0	\n"			\

	: "+ZB" (v->counter)						\

	: "r" (I)							\

	: "memory");							\

}



#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix)			\

static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v)	\

{										\

	long result;								\

	__asm__ __volatile__(							\

	"am"#asm_op#mb".d " " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)					\

	: "r" (I)								\

	: "memory");								\

										\

	return result c_op I;							\

}



#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix)				\

static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v)	\

{										\

	long result;								\

										\

	__asm__ __volatile__(							\

	"am"#asm_op#mb".d " " %1, %2, %0		\n"			\

	: "+ZB" (v->counter), "=&r" (result)					\

	: "r" (I)								\

	: "memory");								\

										\

	return result;								\

}



#define ATOMIC64_OPS(op, I, asm_op, c_op)				      \

	ATOMIC64_OP(op, I, asm_op)					      \

	ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db,         )		      \

	ATOMIC64_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)		      \

	ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )			      \

	ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC64_OPS(add, i, add, +)

ATOMIC64_OPS(sub, -i, add, +)



#define arch_atomic64_add_return		arch_atomic64_add_return

#define arch_atomic64_add_return_acquire	arch_atomic64_add_return

#define arch_atomic64_add_return_release	arch_atomic64_add_return

#define arch_atomic64_add_return_relaxed	arch_atomic64_add_return_relaxed

#define arch_atomic64_sub_return		arch_atomic64_sub_return

#define arch_atomic64_sub_return_acquire	arch_atomic64_sub_return

#define arch_atomic64_sub_return_release	arch_atomic64_sub_return

#define arch_atomic64_sub_return_relaxed	arch_atomic64_sub_return_relaxed

#define arch_atomic64_fetch_add			arch_atomic64_fetch_add

#define arch_atomic64_fetch_add_acquire		arch_atomic64_fetch_add

#define arch_atomic64_fetch_add_release		arch_atomic64_fetch_add

#define arch_atomic64_fetch_add_relaxed		arch_atomic64_fetch_add_relaxed

#define arch_atomic64_fetch_sub			arch_atomic64_fetch_sub

#define arch_atomic64_fetch_sub_acquire		arch_atomic64_fetch_sub

#define arch_atomic64_fetch_sub_release		arch_atomic64_fetch_sub

#define arch_atomic64_fetch_sub_relaxed		arch_atomic64_fetch_sub_relaxed



#undef ATOMIC64_OPS



#define ATOMIC64_OPS(op, I, asm_op)					      \

	ATOMIC64_OP(op, I, asm_op)					      \

	ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )			      \

	ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)



ATOMIC64_OPS(and, i, and)

ATOMIC64_OPS(or, i, or)

ATOMIC64_OPS(xor, i, xor)



#define arch_atomic64_fetch_and		arch_atomic64_fetch_and

#define arch_atomic64_fetch_and_acquire	arch_atomic64_fetch_and

#define arch_atomic64_fetch_and_release	arch_atomic64_fetch_and

#define arch_atomic64_fetch_and_relaxed	arch_atomic64_fetch_and_relaxed

#define arch_atomic64_fetch_or		arch_atomic64_fetch_or

#define arch_atomic64_fetch_or_acquire	arch_atomic64_fetch_or

#define arch_atomic64_fetch_or_release	arch_atomic64_fetch_or

#define arch_atomic64_fetch_or_relaxed	arch_atomic64_fetch_or_relaxed

#define arch_atomic64_fetch_xor		arch_atomic64_fetch_xor

#define arch_atomic64_fetch_xor_acquire	arch_atomic64_fetch_xor

#define arch_atomic64_fetch_xor_release	arch_atomic64_fetch_xor

#define arch_atomic64_fetch_xor_relaxed	arch_atomic64_fetch_xor_relaxed



#undef ATOMIC64_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)

{

       long prev, rc;

arch/loongarch/include/asm/cmpxchg.h

+37 −11
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ 
#include <linux/build_bug.h>

#include <asm/barrier.h>



#define __xchg_asm(amswap_db, m, val)		\

#define __xchg_amo_asm(amswap_db, m, val)	\

({						\

	__typeof(val) __ret;			\

						\
@@ -22,6 +22,22 @@ 
	__ret;					\

})



#define __xchg_llsc_asm(ld, st, m, val)			\

({							\

	__typeof(val) __ret, __tmp;			\

							\

	asm volatile (					\

	"1:	ll.w	%0, %3		\n"		\

	"	move	%1, %z4		\n"		\

	"	sc.w	%1, %2		\n"		\

	"	beqz	%1, 1b		\n"		\

	: "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m)	\

	: "ZC" (*m), "Jr" (val)				\

	: "memory");					\

							\

	__ret;						\

})



static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,

					unsigned int size)

{
@@ -67,13 +83,23 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size) 
	switch (size) {

	case 1:

	case 2:

		return __xchg_small(ptr, x, size);

		return __xchg_small((volatile void *)ptr, x, size);



	case 4:

		return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);

#ifdef CONFIG_CPU_HAS_AMO

		return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);

#else

		return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x);

#endif /* CONFIG_CPU_HAS_AMO */



#ifdef CONFIG_64BIT

	case 8:

		return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);

#ifdef CONFIG_CPU_HAS_AMO

		return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);

#else

		return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x);

#endif /* CONFIG_CPU_HAS_AMO */

#endif /* CONFIG_64BIT */



	default:

		BUILD_BUG();