Commit a3f8a3a2 authored by Uros Bizjak's avatar Uros Bizjak Committed by Ingo Molnar
Browse files

x86/percpu: Rewrite x86_this_cpu_test_bit() and friends as macros 



Rewrite the whole family of x86_this_cpu_test_bit() functions
as macros, so standard __my_cpu_var() and raw_cpu_read() macros
can be used on percpu variables. This approach considerably
simplifies implementation of functions and also introduces
standard checks on accessed percpu variables.

No functional changes intended.

Signed-off-by: default avatarUros Bizjak <ubizjak@gmail.com>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20240404094218.448963-2-ubizjak@gmail.com
parent 4c3677c0

arch/um/include/asm/cpufeature.h

+1 −2
Original line number Diff line number Diff line
@@ -38,8 +38,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32]; 


#define this_cpu_has(bit)						\

	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\

	 x86_this_cpu_test_bit(bit,					\

		(unsigned long __percpu *)&cpu_info.x86_capability))

	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))



/*

 * This macro is for detection of features which need kernel

arch/x86/include/asm/cpufeature.h

+1 −2
Original line number Diff line number Diff line
@@ -127,8 +127,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32]; 


#define this_cpu_has(bit)						\

	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\

	 x86_this_cpu_test_bit(bit,					\

		(unsigned long __percpu *)&cpu_info.x86_capability))

	 x86_this_cpu_test_bit(bit, cpu_info.x86_capability))



/*

 * This macro is for detection of features which need kernel

arch/x86/include/asm/percpu.h

+23 −31
Original line number Diff line number Diff line
@@ -96,7 +96,7 @@ 
#endif /* CONFIG_SMP */



#define __my_cpu_type(var)	typeof(var) __percpu_seg_override

#define __my_cpu_ptr(ptr)	(__my_cpu_type(*ptr)*)(__force uintptr_t)(ptr)

#define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)

#define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))

#define __percpu_arg(x)		__percpu_prefix "%" #x

#define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
@@ -568,37 +568,29 @@ do { \ 
#define this_cpu_read_stable_8(pcp)    ({ BUILD_BUG(); (typeof(pcp))0; })

#endif



static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,

                        const unsigned long __percpu *addr)

{

	unsigned long __percpu *a =

		(unsigned long __percpu *)addr + nr / BITS_PER_LONG;

#define x86_this_cpu_constant_test_bit(_nr, _var)			\

({									\

	unsigned long __percpu *addr__ =				\

		(unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \

	!!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__));	\

})



#ifdef CONFIG_X86_64

	return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0;

#else

	return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0;

#endif

}



static inline bool x86_this_cpu_variable_test_bit(int nr,

                        const unsigned long __percpu *addr)

{

	bool oldbit;



	asm volatile("btl %[nr], " __percpu_arg([var])

			CC_SET(c)

			: CC_OUT(c) (oldbit)

			: [var] "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))),

			  [nr] "Ir" (nr));



	return oldbit;

}



#define x86_this_cpu_test_bit(nr, addr)			\

	(__builtin_constant_p((nr))			\

	 ? x86_this_cpu_constant_test_bit((nr), (addr))	\

	 : x86_this_cpu_variable_test_bit((nr), (addr)))

#define x86_this_cpu_variable_test_bit(_nr, _var)		\

({								\

	bool oldbit;						\

								\

	asm volatile("btl %[nr], " __percpu_arg([var])		\

		     CC_SET(c)					\

		     : CC_OUT(c) (oldbit)			\

		     : [var] "m" (__my_cpu_var(_var)),		\

		       [nr] "rI" (_nr));			\

	oldbit;							\

})



#define x86_this_cpu_test_bit(_nr, _var)			\

	(__builtin_constant_p(_nr)				\

	 ? x86_this_cpu_constant_test_bit(_nr, _var)		\

	 : x86_this_cpu_variable_test_bit(_nr, _var))





#include <asm-generic/percpu.h>