Commit 3c302e14 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'x86-cfi-bpf-fix-cfi-vs-ebpf'

Peter Zijlstra says:

====================
x86/cfi,bpf: Fix CFI vs eBPF

Hi!

What started with the simple observation that bpf_dispatcher_*_func() was
broken for calling CFI functions with a __nocfi calling context for FineIBT
ended up with a complete BPF wide CFI fixup.

With these changes on the BPF selftest suite passes without crashing -- there's
still a few failures, but Alexei has graciously offered to look into those.

(Alexei, I have presumed your SoB on the very last patch, please update
as you see fit)

Changes since v2 are numerous but include:
 - cfi_get_offset() -- as a means to communicate the offset (ast)
 - 5 new patches fixing various BPF internals to be CFI clean

Note: it *might* be possible to merge the
bpf_bpf_tcp_ca.c:unsupported_ops[] thing into the CFI stubs, as is
get_info will have a NULL stub, unlike the others.
---
 arch/riscv/include/asm/cfi.h   |   3 +-
 arch/riscv/kernel/cfi.c        |   2 +-
 arch/x86/include/asm/cfi.h     | 126 +++++++++++++++++++++++++++++++++++++-
 arch/x86/kernel/alternative.c  |  87 +++++++++++++++++++++++---
 arch/x86/kernel/cfi.c          |   4 +-
 arch/x86/net/bpf_jit_comp.c    | 134 +++++++++++++++++++++++++++++++++++------
 include/asm-generic/Kbuild     |   1 +
 include/linux/bpf.h            |  27 ++++++++-
 include/linux/cfi.h            |  12 ++++
 kernel/bpf/bpf_struct_ops.c    |  16 ++---
 kernel/bpf/core.c              |  25 ++++++++
 kernel/bpf/cpumask.c           |   8 ++-
 kernel/bpf/helpers.c           |  18 +++++-
 net/bpf/bpf_dummy_struct_ops.c |  31 +++++++++-
 net/bpf/test_run.c             |  15 ++++-
 net/ipv4/bpf_tcp_ca.c          |  69 +++++++++++++++++++++
 16 files changed, 528 insertions(+), 50 deletions(-)
====================

Link: https://lore.kernel.org/r/20231215091216.135791411@infradead.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 1467affd 852486b3
Loading
Loading
Loading
Loading
+2 −1
Original line number Diff line number Diff line
@@ -7,8 +7,9 @@
 *
 * Copyright (C) 2023 Google LLC
 */
#include <linux/bug.h>

#include <linux/cfi.h>
struct pt_regs;

#ifdef CONFIG_CFI_CLANG
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
+1 −1
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@
 *
 * Copyright (C) 2023 Google LLC
 */
#include <asm/cfi.h>
#include <linux/cfi.h>
#include <asm/insn.h>

/*
+125 −1
Original line number Diff line number Diff line
@@ -7,16 +7,140 @@
 *
 * Copyright (C) 2022 Google LLC
 */
#include <linux/bug.h>
#include <asm/ibt.h>

#include <linux/cfi.h>
/*
 * An overview of the various calling conventions...
 *
 * Traditional:
 *
 * foo:
 *   ... code here ...
 *   ret
 *
 * direct caller:
 *   call foo
 *
 * indirect caller:
 *   lea foo(%rip), %r11
 *   ...
 *   call *%r11
 *
 *
 * IBT:
 *
 * foo:
 *   endbr64
 *   ... code here ...
 *   ret
 *
 * direct caller:
 *   call foo / call foo+4
 *
 * indirect caller:
 *   lea foo(%rip), %r11
 *   ...
 *   call *%r11
 *
 *
 * kCFI:
 *
 * __cfi_foo:
 *   movl $0x12345678, %eax
 *				# 11 nops when CONFIG_CALL_PADDING
 * foo:
 *   endbr64			# when IBT
 *   ... code here ...
 *   ret
 *
 * direct call:
 *   call foo			# / call foo+4 when IBT
 *
 * indirect call:
 *   lea foo(%rip), %r11
 *   ...
 *   movl $(-0x12345678), %r10d
 *   addl -4(%r11), %r10d	# -15 when CONFIG_CALL_PADDING
 *   jz   1f
 *   ud2
 * 1:call *%r11
 *
 *
 * FineIBT (builds as kCFI + CALL_PADDING + IBT + RETPOLINE and runtime patches into):
 *
 * __cfi_foo:
 *   endbr64
 *   subl 0x12345678, %r10d
 *   jz   foo
 *   ud2
 *   nop
 * foo:
 *   osp nop3			# was endbr64
 *   ... code here ...
 *   ret
 *
 * direct caller:
 *   call foo / call foo+4
 *
 * indirect caller:
 *   lea foo(%rip), %r11
 *   ...
 *   movl $0x12345678, %r10d
 *   subl $16, %r11
 *   nop4
 *   call *%r11
 *
 */
enum cfi_mode {
	CFI_DEFAULT,	/* FineIBT if hardware has IBT, otherwise kCFI */
	CFI_OFF,	/* Taditional / IBT depending on .config */
	CFI_KCFI,	/* Optionally CALL_PADDING, IBT, RETPOLINE */
	CFI_FINEIBT,	/* see arch/x86/kernel/alternative.c */
};

extern enum cfi_mode cfi_mode;

struct pt_regs;

#ifdef CONFIG_CFI_CLANG
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
#define __bpfcall
extern u32 cfi_bpf_hash;
extern u32 cfi_bpf_subprog_hash;

static inline int cfi_get_offset(void)
{
	switch (cfi_mode) {
	case CFI_FINEIBT:
		return 16;
	case CFI_KCFI:
		if (IS_ENABLED(CONFIG_CALL_PADDING))
			return 16;
		return 5;
	default:
		return 0;
	}
}
#define cfi_get_offset cfi_get_offset

extern u32 cfi_get_func_hash(void *func);

#else
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
{
	return BUG_TRAP_TYPE_NONE;
}
#define cfi_bpf_hash 0U
#define cfi_bpf_subprog_hash 0U
static inline u32 cfi_get_func_hash(void *func)
{
	return 0;
}
#endif /* CONFIG_CFI_CLANG */

#if HAS_KERNEL_IBT == 1
#define CFI_NOSEAL(x)	asm(IBT_NOSEAL(__stringify(x)))
#endif

#endif /* _ASM_X86_CFI_H */
+79 −8
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@
#include <asm/fixmap.h>
#include <asm/paravirt.h>
#include <asm/asm-prototypes.h>
#include <asm/cfi.h>

int __read_mostly alternatives_patched;

@@ -832,15 +833,82 @@ void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }
#endif /* CONFIG_X86_KERNEL_IBT */

#ifdef CONFIG_FINEIBT
#define __CFI_DEFAULT	CFI_DEFAULT
#elif defined(CONFIG_CFI_CLANG)
#define __CFI_DEFAULT	CFI_KCFI
#else
#define __CFI_DEFAULT	CFI_OFF
#endif

enum cfi_mode {
	CFI_DEFAULT,
	CFI_OFF,
	CFI_KCFI,
	CFI_FINEIBT,
};
enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;

#ifdef CONFIG_CFI_CLANG
struct bpf_insn;

/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
extern unsigned int __bpf_prog_runX(const void *ctx,
				    const struct bpf_insn *insn);

/*
 * Force a reference to the external symbol so the compiler generates
 * __kcfi_typid.
 */
__ADDRESSABLE(__bpf_prog_runX);

/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
asm (
"	.pushsection	.data..ro_after_init,\"aw\",@progbits	\n"
"	.type	cfi_bpf_hash,@object				\n"
"	.globl	cfi_bpf_hash					\n"
"	.p2align	2, 0x0					\n"
"cfi_bpf_hash:							\n"
"	.long	__kcfi_typeid___bpf_prog_runX			\n"
"	.size	cfi_bpf_hash, 4					\n"
"	.popsection						\n"
);

/* Must match bpf_callback_t */
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);

__ADDRESSABLE(__bpf_callback_fn);

/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
asm (
"	.pushsection	.data..ro_after_init,\"aw\",@progbits	\n"
"	.type	cfi_bpf_subprog_hash,@object			\n"
"	.globl	cfi_bpf_subprog_hash				\n"
"	.p2align	2, 0x0					\n"
"cfi_bpf_subprog_hash:						\n"
"	.long	__kcfi_typeid___bpf_callback_fn			\n"
"	.size	cfi_bpf_subprog_hash, 4				\n"
"	.popsection						\n"
);

u32 cfi_get_func_hash(void *func)
{
	u32 hash;

	func -= cfi_get_offset();
	switch (cfi_mode) {
	case CFI_FINEIBT:
		func += 7;
		break;
	case CFI_KCFI:
		func += 1;
		break;
	default:
		return 0;
	}

	if (get_kernel_nofault(hash, func))
		return 0;

	return hash;
}
#endif

#ifdef CONFIG_FINEIBT

static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT;
static bool cfi_rand __ro_after_init = true;
static u32  cfi_seed __ro_after_init;

@@ -1149,8 +1217,11 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
		goto err;

	if (cfi_rand) {
		if (builtin)
		if (builtin) {
			cfi_seed = get_random_u32();
			cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
			cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
		}

		ret = cfi_rand_preamble(start_cfi, end_cfi);
		if (ret)
+2 −2
Original line number Diff line number Diff line
@@ -4,10 +4,10 @@
 *
 * Copyright (C) 2022 Google LLC
 */
#include <asm/cfi.h>
#include <linux/string.h>
#include <linux/cfi.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
#include <linux/string.h>

/*
 * Returns the target address and the expected type when regs->ip points
Loading