Commit aecaf181 authored by Alice Ryhl's avatar Alice Ryhl Committed by Steven Rostedt (Google)
Browse files

jump_label: adjust inline asm to be consistent 

To avoid duplication of inline asm between C and Rust, we need to
import the inline asm from the relevant `jump_label.h` header into Rust.
To make that easier, this patch updates the header files to expose the
inline asm via a new ARCH_STATIC_BRANCH_ASM macro.

The header files are all updated to define a ARCH_STATIC_BRANCH_ASM that
takes the same arguments in a consistent order so that Rust can use the
same logic for every architecture.

Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Josh Poimboeuf <jpoimboe@kernel.org>
Cc: Jason Baron <jbaron@akamai.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Gary Guo <gary@garyguo.net>
Cc: " =?utf-8?q?Bj=C3=B6rn_Roy_Baron?= " <bjorn3_gh@protonmail.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: Andreas Hindborg <a.hindborg@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: Uros Bizjak <ubizjak@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Oliver Upton <oliver.upton@linux.dev>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Fuad Tabba <tabba@google.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Anup Patel <apatel@ventanamicro.com>
Cc: Andrew Jones <ajones@ventanamicro.com>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: Conor Dooley <conor.dooley@microchip.com>
Cc: Samuel Holland <samuel.holland@sifive.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Bibo Mao <maobibo@loongson.cn>
Cc: Tiezhu Yang <yangtiezhu@loongson.cn>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Tianrui Zhao <zhaotianrui@loongson.cn>
Cc: Palmer Dabbelt <palmer@rivosinc.com>
Link: https://lore.kernel.org/20241030-tracepoint-v12-4-eec7f0f8ad22@google.com


Suggested-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Co-developed-by: default avatarMiguel Ojeda <ojeda@kernel.org>
Signed-off-by: default avatarMiguel Ojeda <ojeda@kernel.org>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Acked-by: Palmer Dabbelt <palmer@rivosinc.com> # RISC-V
Signed-off-by: default avatarAlice Ryhl <aliceryhl@google.com>
Signed-off-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
parent 91d39024

arch/arm/include/asm/jump_label.h

+9 −5
Original line number Diff line number Diff line
@@ -9,13 +9,17 @@ 


#define JUMP_LABEL_NOP_SIZE 4



/* This macro is also expanded on the Rust side. */

#define ARCH_STATIC_BRANCH_ASM(key, label)		\

	"1:\n\t"					\

	WASM(nop) "\n\t"				\

	".pushsection __jump_table,  \"aw\"\n\t"	\

	".word 1b, " label ", " key "\n\t"		\

	".popsection\n\t"				\



static __always_inline bool arch_static_branch(struct static_key *key, bool branch)

{

	asm goto("1:\n\t"

		 WASM(nop) "\n\t"

		 ".pushsection __jump_table,  \"aw\"\n\t"

		 ".word 1b, %l[l_yes], %c0\n\t"

		 ".popsection\n\t"

	asm goto(ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]")

		 : :  "i" (&((char *)key)[branch]) :  : l_yes);



	return false;

arch/arm64/include/asm/jump_label.h

+13 −7
Original line number Diff line number Diff line
@@ -19,10 +19,14 @@ 
#define JUMP_TABLE_ENTRY(key, label)			\

	".pushsection	__jump_table, \"aw\"\n\t"	\

	".align		3\n\t"				\

	".long		1b - ., %l["#label"] - .\n\t"	\

	".quad		%c0 - .\n\t"			\

	".popsection\n\t"				\

	:  :  "i"(key) :  : label

	".long		1b - ., " label " - .\n\t"	\

	".quad		" key " - .\n\t"		\

	".popsection\n\t"



/* This macro is also expanded on the Rust side. */

#define ARCH_STATIC_BRANCH_ASM(key, label)		\

	"1:	nop\n\t"				\

	JUMP_TABLE_ENTRY(key, label)



static __always_inline bool arch_static_branch(struct static_key * const key,

					       const bool branch)
@@ -30,8 +34,8 @@ static __always_inline bool arch_static_branch(struct static_key * const key, 
	char *k = &((char *)key)[branch];



	asm goto(

		"1:	nop					\n\t"

		JUMP_TABLE_ENTRY(k, l_yes)

		ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]")

		:  :  "i"(k) :  : l_yes

		);



	return false;
@@ -43,9 +47,11 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke 
						    const bool branch)

{

	char *k = &((char *)key)[branch];



	asm goto(

		"1:	b		%l[l_yes]		\n\t"

		JUMP_TABLE_ENTRY(k, l_yes)

		JUMP_TABLE_ENTRY("%c0", "%l[l_yes]")

		:  :  "i"(k) :  : l_yes

		);

	return false;

l_yes:

arch/loongarch/include/asm/jump_label.h

+10 −6
Original line number Diff line number Diff line
@@ -13,18 +13,22 @@ 


#define JUMP_LABEL_NOP_SIZE	4



#define JUMP_TABLE_ENTRY				\

/* This macro is also expanded on the Rust side. */

#define JUMP_TABLE_ENTRY(key, label)			\

	 ".pushsection	__jump_table, \"aw\"	\n\t"	\

	 ".align	3			\n\t"	\

	 ".long		1b - ., %l[l_yes] - .	\n\t"	\

	 ".quad		%0 - .			\n\t"	\

	 ".long		1b - ., " label " - .	\n\t"	\

	 ".quad		" key " - .		\n\t"	\

	 ".popsection				\n\t"



#define ARCH_STATIC_BRANCH_ASM(key, label)		\

	"1:	nop				\n\t"	\

	JUMP_TABLE_ENTRY(key, label)



static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)

{

	asm goto(

		"1:	nop			\n\t"

		JUMP_TABLE_ENTRY

		ARCH_STATIC_BRANCH_ASM("%0", "%l[l_yes]")

		:  :  "i"(&((char *)key)[branch]) :  : l_yes);



	return false;
@@ -37,7 +41,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke 
{

	asm goto(

		"1:	b	%l[l_yes]	\n\t"

		JUMP_TABLE_ENTRY

		JUMP_TABLE_ENTRY("%0", "%l[l_yes]")

		:  :  "i"(&((char *)key)[branch]) :  : l_yes);



	return false;

arch/riscv/include/asm/jump_label.h

+28 −22
Original line number Diff line number Diff line
@@ -16,21 +16,28 @@ 


#define JUMP_LABEL_NOP_SIZE 4



#define JUMP_TABLE_ENTRY(key, label)			\

	".pushsection	__jump_table, \"aw\"	\n\t"	\

	".align		" RISCV_LGPTR "		\n\t"	\

	".long		1b - ., " label " - .	\n\t"	\

	"" RISCV_PTR "	" key " - .		\n\t"	\

	".popsection				\n\t"



/* This macro is also expanded on the Rust side. */

#define ARCH_STATIC_BRANCH_ASM(key, label)		\

	"	.align		2		\n\t"	\

	"	.option push			\n\t"	\

	"	.option norelax			\n\t"	\

	"	.option norvc			\n\t"	\

	"1:	nop				\n\t"	\

	"	.option pop			\n\t"	\

	JUMP_TABLE_ENTRY(key, label)



static __always_inline bool arch_static_branch(struct static_key * const key,

					       const bool branch)

{

	asm goto(

		"	.align		2			\n\t"

		"	.option push				\n\t"

		"	.option norelax				\n\t"

		"	.option norvc				\n\t"

		"1:	nop					\n\t"

		"	.option pop				\n\t"

		"	.pushsection	__jump_table, \"aw\"	\n\t"

		"	.align		" RISCV_LGPTR "		\n\t"

		"	.long		1b - ., %l[label] - .	\n\t"

		"	" RISCV_PTR "	%0 - .			\n\t"

		"	.popsection				\n\t"

		ARCH_STATIC_BRANCH_ASM("%0", "%l[label]")

		:  :  "i"(&((char *)key)[branch]) :  : label);



	return false;
@@ -38,21 +45,20 @@ static __always_inline bool arch_static_branch(struct static_key * const key, 
	return true;

}



#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label)		\

	"	.align		2		\n\t"	\

	"	.option push			\n\t"	\

	"	.option norelax			\n\t"	\

	"	.option norvc			\n\t"	\

	"1:	j	" label "		\n\t" \

	"	.option pop			\n\t"	\

	JUMP_TABLE_ENTRY(key, label)



static __always_inline bool arch_static_branch_jump(struct static_key * const key,

						    const bool branch)

{

	asm goto(

		"	.align		2			\n\t"

		"	.option push				\n\t"

		"	.option norelax				\n\t"

		"	.option norvc				\n\t"

		"1:	j		%l[label]		\n\t"

		"	.option pop				\n\t"

		"	.pushsection	__jump_table, \"aw\"	\n\t"

		"	.align		" RISCV_LGPTR "		\n\t"

		"	.long		1b - ., %l[label] - .	\n\t"

		"	" RISCV_PTR "	%0 - .			\n\t"

		"	.popsection				\n\t"

		ARCH_STATIC_BRANCH_JUMP_ASM("%0", "%l[label]")

		:  :  "i"(&((char *)key)[branch]) :  : label);



	return false;

arch/x86/include/asm/jump_label.h

+13 −22
Original line number Diff line number Diff line
@@ -12,35 +12,28 @@ 
#include <linux/stringify.h>

#include <linux/types.h>



#define JUMP_TABLE_ENTRY				\

#define JUMP_TABLE_ENTRY(key, label)			\

	".pushsection __jump_table,  \"aw\" \n\t"	\

	_ASM_ALIGN "\n\t"				\

	".long 1b - . \n\t"				\

	".long %l[l_yes] - . \n\t"			\

	_ASM_PTR "%c0 + %c1 - .\n\t"			\

	".long " label " - . \n\t"			\

	_ASM_PTR " " key " - . \n\t"			\

	".popsection \n\t"



/* This macro is also expanded on the Rust side. */

#ifdef CONFIG_HAVE_JUMP_LABEL_HACK



static __always_inline bool arch_static_branch(struct static_key *key, bool branch)

{

	asm goto("1:"

		"jmp %l[l_yes] # objtool NOPs this \n\t"

		JUMP_TABLE_ENTRY

		: :  "i" (key), "i" (2 | branch) : : l_yes);



	return false;

l_yes:

	return true;

}



#define ARCH_STATIC_BRANCH_ASM(key, label)		\

	"1: jmp " label " # objtool NOPs this \n\t"	\

	JUMP_TABLE_ENTRY(key " + 2", label)

#else /* !CONFIG_HAVE_JUMP_LABEL_HACK */

#define ARCH_STATIC_BRANCH_ASM(key, label)		\

	"1: .byte " __stringify(BYTES_NOP5) "\n\t"	\

	JUMP_TABLE_ENTRY(key, label)

#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */



static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)

{

	asm goto("1:"

		".byte " __stringify(BYTES_NOP5) "\n\t"

		JUMP_TABLE_ENTRY

	asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]")

		: :  "i" (key), "i" (branch) : : l_yes);



	return false;
@@ -48,13 +41,11 @@ static __always_inline bool arch_static_branch(struct static_key * const key, co 
	return true;

}



#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */



static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)

{

	asm goto("1:"

		"jmp %l[l_yes]\n\t"

		JUMP_TABLE_ENTRY

		JUMP_TABLE_ENTRY("%c0 + %c1", "%l[l_yes]")

		: :  "i" (key), "i" (branch) : : l_yes);



	return false;