Commit 4bebb991 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-arm64-add-fsession-support' 

Leon Hwang says:

====================
Similar to commit 98770bd4 ("bpf,x86: add fsession support for x86_64"),
add fsession support on arm64.

Patch #1 adds bpf_jit_supports_fsession() to prevent fsession loading
on architectures that do not implement fsession support.

Patch #2 implements fsession support in the arm64 BPF JIT trampoline.

Patch #3 enables the relevant selftests on arm64, including get_func_ip,
and get_func_args.

All enabled tests pass on arm64:

 cd tools/testing/selftests/bpf
 ./test_progs -t fsession
 #136/1   fsession_test/fsession_test:OK
 #136/2   fsession_test/fsession_reattach:OK
 #136/3   fsession_test/fsession_cookie:OK
 #136     fsession_test:OK
 Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILED

 ./test_progs -t get_func
 #138     get_func_args_test:OK
 #139     get_func_ip_test:OK
 Summary: 2/0 PASSED, 0 SKIPPED, 0 FAILED

Changes:
v4 -> v5:
* Address comment from Alexei:
  * Rename helper bpf_link_prog_session_cookie() to
    bpf_prog_calls_session_cookie().
* v4: https://lore.kernel.org/bpf/20260129154953.66915-1-leon.hwang@linux.dev/

v3 -> v4:
* Add a log when !bpf_jit_supports_fsession() in patch #1 (per AI).
* v3: https://lore.kernel.org/bpf/20260129142536.48637-1-leon.hwang@linux.dev/

v2 -> v3:
* Fix typo in subject and patch message of patch #1 (per AI and Chris).
* Collect Acked-by, and Tested-by from Puranjay, thanks.
* v2: https://lore.kernel.org/bpf/20260128150112.8873-1-leon.hwang@linux.dev/

v1 -> v2:
* Add bpf_jit_supports_fsession().
* v1: https://lore.kernel.org/bpf/20260127163344.92819-1-leon.hwang@linux.dev/
====================

Link: https://patch.msgid.link/20260131144950.16294-1-leon.hwang@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents f0b5b3d6 7f10da21

arch/arm64/net/bpf_jit_comp.c

+62 −9
Original line number Diff line number Diff line
@@ -2510,6 +2510,12 @@ static bool is_struct_ops_tramp(const struct bpf_tramp_links *fentry_links) 
		fentry_links->links[0]->link.type == BPF_LINK_TYPE_STRUCT_OPS;

}



static void store_func_meta(struct jit_ctx *ctx, u64 func_meta, int func_meta_off)

{

	emit_a64_mov_i64(A64_R(10), func_meta, ctx);

	emit(A64_STR64I(A64_R(10), A64_SP, func_meta_off), ctx);

}



/* Based on the x86's implementation of arch_prepare_bpf_trampoline().

 *

 * bpf prog and function entry before bpf trampoline hooked:
@@ -2533,7 +2539,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
	int regs_off;

	int retval_off;

	int bargs_off;

	int nfuncargs_off;

	int func_meta_off;

	int ip_off;

	int run_ctx_off;

	int oargs_off;
@@ -2544,6 +2550,9 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
	bool save_ret;

	__le32 **branches = NULL;

	bool is_struct_ops = is_struct_ops_tramp(fentry);

	int cookie_off, cookie_cnt, cookie_bargs_off;

	int fsession_cnt = bpf_fsession_cnt(tlinks);

	u64 func_meta;



	/* trampoline stack layout:

	 *                    [ parent ip         ]
@@ -2562,10 +2571,14 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
	 *                    [ ...               ]

	 * SP + bargs_off     [ arg reg 1         ] for bpf

	 *

	 * SP + nfuncargs_off [ arg regs count    ]

	 * SP + func_meta_off [ regs count, etc   ]

	 *

	 * SP + ip_off        [ traced function   ] BPF_TRAMP_F_IP_ARG flag

	 *

	 *                    [ stack cookie N    ]

	 *                    [ ...               ]

	 * SP + cookie_off    [ stack cookie 1    ]

	 *

	 * SP + run_ctx_off   [ bpf_tramp_run_ctx ]

	 *

	 *                    [ stack arg N       ]
@@ -2582,13 +2595,18 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
	/* room for bpf_tramp_run_ctx */

	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);



	cookie_off = stack_size;

	/* room for session cookies */

	cookie_cnt = bpf_fsession_cookie_cnt(tlinks);

	stack_size += cookie_cnt * 8;



	ip_off = stack_size;

	/* room for IP address argument */

	if (flags & BPF_TRAMP_F_IP_ARG)

		stack_size += 8;



	nfuncargs_off = stack_size;

	/* room for args count */

	func_meta_off = stack_size;

	/* room for function metadata, such as regs count */

	stack_size += 8;



	bargs_off = stack_size;
@@ -2646,9 +2664,9 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
		emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx);

	}



	/* save arg regs count*/

	emit(A64_MOVZ(1, A64_R(10), nfuncargs, 0), ctx);

	emit(A64_STR64I(A64_R(10), A64_SP, nfuncargs_off), ctx);

	/* save function metadata */

	func_meta = nfuncargs;

	store_func_meta(ctx, func_meta, func_meta_off);



	/* save args for bpf */

	save_args(ctx, bargs_off, oargs_off, m, a, false);
@@ -2666,10 +2684,27 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
		emit_call((const u64)__bpf_tramp_enter, ctx);

	}



	for (i = 0; i < fentry->nr_links; i++)

	if (fsession_cnt) {

		/* clear all the session cookies' value */

		emit(A64_MOVZ(1, A64_R(10), 0, 0), ctx);

		for (int i = 0; i < cookie_cnt; i++)

			emit(A64_STR64I(A64_R(10), A64_SP, cookie_off + 8 * i), ctx);

		/* clear the return value to make sure fentry always gets 0 */

		emit(A64_STR64I(A64_R(10), A64_SP, retval_off), ctx);

	}



	cookie_bargs_off = (bargs_off - cookie_off) / 8;

	for (i = 0; i < fentry->nr_links; i++) {

		if (bpf_prog_calls_session_cookie(fentry->links[i])) {

			u64 meta = func_meta | (cookie_bargs_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);



			store_func_meta(ctx, meta, func_meta_off);

			cookie_bargs_off--;

		}

		invoke_bpf_prog(ctx, fentry->links[i], bargs_off,

				retval_off, run_ctx_off,

				flags & BPF_TRAMP_F_RET_FENTRY_RET);

	}



	if (fmod_ret->nr_links) {

		branches = kcalloc(fmod_ret->nr_links, sizeof(__le32 *),
@@ -2701,9 +2736,22 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
		*branches[i] = cpu_to_le32(A64_CBNZ(1, A64_R(10), offset));

	}



	for (i = 0; i < fexit->nr_links; i++)

	/* set the "is_return" flag for fsession */

	func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);

	if (fsession_cnt)

		store_func_meta(ctx, func_meta, func_meta_off);



	cookie_bargs_off = (bargs_off - cookie_off) / 8;

	for (i = 0; i < fexit->nr_links; i++) {

		if (bpf_prog_calls_session_cookie(fexit->links[i])) {

			u64 meta = func_meta | (cookie_bargs_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);



			store_func_meta(ctx, meta, func_meta_off);

			cookie_bargs_off--;

		}

		invoke_bpf_prog(ctx, fexit->links[i], bargs_off, retval_off,

				run_ctx_off, false);

	}



	if (flags & BPF_TRAMP_F_CALL_ORIG) {

		im->ip_epilogue = ctx->ro_image + ctx->idx;
@@ -2753,6 +2801,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
	return ctx->idx;

}



bool bpf_jit_supports_fsession(void)

{

	return true;

}



int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

			     struct bpf_tramp_links *tlinks, void *func_addr)

{

arch/x86/net/bpf_jit_comp.c

+5 −0
Original line number Diff line number Diff line
@@ -4112,3 +4112,8 @@ bool bpf_jit_supports_timed_may_goto(void) 
{

	return true;

}



bool bpf_jit_supports_fsession(void)

{

	return true;

}

include/linux/bpf.h

+6 −1
Original line number Diff line number Diff line
@@ -2196,13 +2196,18 @@ static inline int bpf_fsession_cnt(struct bpf_tramp_links *links) 
	return cnt;

}



static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_link *link)

{

	return link->link.prog->call_session_cookie;

}



static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_links *links)

{

	struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];

	int cnt = 0;



	for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {

		if (fentries.links[i]->link.prog->call_session_cookie)

		if (bpf_prog_calls_session_cookie(fentries.links[i]))

			cnt++;

	}

include/linux/filter.h

+1 −0
Original line number Diff line number Diff line
@@ -1167,6 +1167,7 @@ bool bpf_jit_supports_arena(void); 
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);

bool bpf_jit_supports_private_stack(void);

bool bpf_jit_supports_timed_may_goto(void);

bool bpf_jit_supports_fsession(void);

u64 bpf_arch_uaddress_limit(void);

void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);

u64 arch_bpf_timed_may_goto(void);

kernel/bpf/core.c

+5 −0
Original line number Diff line number Diff line
@@ -3144,6 +3144,11 @@ bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 
	return false;

}



bool __weak bpf_jit_supports_fsession(void)

{

	return false;

}



u64 __weak bpf_arch_uaddress_limit(void)

{

#if defined(CONFIG_64BIT) && defined(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE)