Commit 54ac9ff8 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Catalin Marinas
Browse files

arm64: Use static call trampolines when kCFI is enabled 



Implement arm64 support for the 'unoptimized' static call variety, which
routes all calls through a trampoline that performs a tail call to the
chosen function, and wire it up for use when kCFI is enabled. This works
around an issue with kCFI and generic static calls, where the prototypes
of default handlers such as __static_call_nop() and __static_call_ret0()
don't match the expected prototype of the call site, resulting in kCFI
false positives [0].

Since static call targets may be located in modules loaded out of direct
branching range, this needs an ADRP/LDR pair to load the branch target
into R16 and a branch-to-register (BR) instruction to perform an
indirect call.

Unlike on x86, there is no pressing need on arm64 to avoid indirect
calls at all cost, but hiding it from the compiler as is done here does
have some benefits:
- the literal is located in .rodata, which gives us the same robustness
  advantage that code patching does;
- no D-cache pollution from fetching hash values from .text sections.

From an execution speed PoV, this is unlikely to make any difference at
all.

Cc: Sami Tolvanen <samitolvanen@google.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: Kees Cook <kees@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will McVicker <willmcvicker@google.com>
Reported-by: default avatarCarlos Llamas <cmllamas@google.com>
Closes: https://lore.kernel.org/all/20260311225822.1565895-1-cmllamas@google.com/

 [0]
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 8c6e9b60

arch/arm64/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -252,6 +252,7 @@ config ARM64 
	select HAVE_RSEQ

	select HAVE_RUST if RUSTC_SUPPORTS_ARM64

	select HAVE_STACKPROTECTOR

	select HAVE_STATIC_CALL if CFI

	select HAVE_SYSCALL_TRACEPOINTS

	select HAVE_KPROBES

	select HAVE_KRETPROBES

arch/arm64/include/asm/static_call.h

0 → 100644
+31 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _ASM_STATIC_CALL_H

#define _ASM_STATIC_CALL_H



#define __ARCH_DEFINE_STATIC_CALL_TRAMP(name, target)		    \

	asm("	.pushsection .static_call.text, \"ax\"		\n" \

	    "	.align	4					\n" \

	    "	.globl	" name "				\n" \

	    name ":						\n" \

	    "	hint	34	/* BTI C */			\n" \

	    "	adrp	x16, 1f					\n" \

	    "	ldr	x16, [x16, :lo12:1f]			\n" \

	    "	br	x16					\n" \

	    "	.type	" name ", %function			\n" \

	    "	.size	" name ", . - " name "			\n" \

	    "	.popsection					\n" \

	    "	.pushsection .rodata, \"a\"			\n" \

	    "	.align	3					\n" \

	    "1:	.quad	" target "				\n" \

	    "	.popsection					\n")



#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)			\

	__ARCH_DEFINE_STATIC_CALL_TRAMP(STATIC_CALL_TRAMP_STR(name), #func)



#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)			\

	ARCH_DEFINE_STATIC_CALL_TRAMP(name, __static_call_return0)



#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)			\

	ARCH_DEFINE_STATIC_CALL_TRAMP(name, __static_call_return0)



#endif /* _ASM_STATIC_CALL_H */

arch/arm64/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ obj-$(CONFIG_MODULES) += module.o module-plts.o 
obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o perf_callchain.o

obj-$(CONFIG_HARDLOCKUP_DETECTOR_PERF)	+= watchdog_hld.o

obj-$(CONFIG_HAVE_HW_BREAKPOINT)	+= hw_breakpoint.o

obj-$(CONFIG_HAVE_STATIC_CALL)		+= static_call.o

obj-$(CONFIG_CPU_PM)			+= sleep.o suspend.o

obj-$(CONFIG_KGDB)			+= kgdb.o

obj-$(CONFIG_EFI)			+= efi.o efi-rt-wrapper.o

arch/arm64/kernel/static_call.c

0 → 100644
+23 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

#include <linux/static_call.h>

#include <linux/memory.h>

#include <asm/text-patching.h>



void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)

{

	u64 literal;

	int ret;



	if (!func)

		func = __static_call_return0;



	/* decode the instructions to discover the literal address */

	literal = ALIGN_DOWN((u64)tramp + 4, SZ_4K) +

		  aarch64_insn_adrp_get_offset(le32_to_cpup(tramp + 4)) +

		  8 * aarch64_insn_decode_immediate(AARCH64_INSN_IMM_12,

						    le32_to_cpup(tramp + 8));



	ret = aarch64_insn_write_literal_u64((void *)literal, (u64)func);

	WARN_ON_ONCE(ret);

}

EXPORT_SYMBOL_GPL(arch_static_call_transform);

arch/arm64/kernel/vmlinux.lds.S

+1 −0
Original line number Diff line number Diff line
@@ -191,6 +191,7 @@ SECTIONS 
			LOCK_TEXT

			KPROBES_TEXT

			HYPERVISOR_TEXT

			STATIC_CALL_TEXT

			*(.gnu.warning)

	}