Commit 5b3fb8a6 authored by Ryan Roberts's avatar Ryan Roberts Committed by Catalin Marinas
Browse files

arm64: mm: Re-implement the __tlbi_level macro as a C function 



As part of efforts to reduce our reliance on complex preprocessor macros
for TLB invalidation routines, convert the __tlbi_level macro to a C
function for by-level TLB invalidation.

Each specific tlbi level op is implemented as a C function and the
appropriate function pointer is passed to __tlbi_level(). Since
everything is declared inline and is statically resolvable, the compiler
will convert the indirect function call to a direct inline execution.

Suggested-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: default avatarJonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: default avatarRyan Roberts <ryan.roberts@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 1f318b96

arch/arm64/include/asm/tlbflush.h

+54 −13
Original line number Diff line number Diff line
@@ -97,19 +97,60 @@ static inline unsigned long get_trans_granule(void) 


#define TLBI_TTL_UNKNOWN	INT_MAX



#define __tlbi_level(op, addr, level) do {				\

	u64 arg = addr;							\

									\

	if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) &&	\

	    level >= 0 && level <= 3) {					\

		u64 ttl = level & 3;					\

		ttl |= get_trans_granule() << 2;			\

		arg &= ~TLBI_TTL_MASK;					\

		arg |= FIELD_PREP(TLBI_TTL_MASK, ttl);			\

	}								\

									\

	__tlbi(op, arg);						\

} while(0)

typedef void (*tlbi_op)(u64 arg);



static __always_inline void vae1is(u64 arg)

{

	__tlbi(vae1is, arg);

}



static __always_inline void vae2is(u64 arg)

{

	__tlbi(vae2is, arg);

}



static __always_inline void vale1(u64 arg)

{

	__tlbi(vale1, arg);

}



static __always_inline void vale1is(u64 arg)

{

	__tlbi(vale1is, arg);

}



static __always_inline void vale2is(u64 arg)

{

	__tlbi(vale2is, arg);

}



static __always_inline void vaale1is(u64 arg)

{

	__tlbi(vaale1is, arg);

}



static __always_inline void ipas2e1(u64 arg)

{

	__tlbi(ipas2e1, arg);

}



static __always_inline void ipas2e1is(u64 arg)

{

	__tlbi(ipas2e1is, arg);

}



static __always_inline void __tlbi_level(tlbi_op op, u64 addr, u32 level)

{

	u64 arg = addr;



	if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) {

		u64 ttl = level | (get_trans_granule() << 2);



		FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl);

	}



	op(arg);

}



#define __tlbi_user_level(op, arg, level) do {				\

	if (arm64_kernel_unmapped_at_el0())				\