Commit bd27626f authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-arm64-support-for-timed-may_goto' 

Puranjay Mohan says:

====================
bpf, arm64: support for timed may_goto

Changes in v2->v3:
v2: https://lore.kernel.org/all/20250809204833.44803-1-puranjay@kernel.org/
- Rebased on bpf-next/master
- Added Acked-by: tags from Xu and Kumar

Changes in v1->v2:
v1: https://lore.kernel.org/bpf/20250724125443.26182-1-puranjay@kernel.org/
- Added comment in arch_bpf_timed_may_goto() about BPF_REG_FP setup (Xu
  Kuohai)

This set adds support for the timed may_goto instruction for the arm64.
The timed may_goto instruction is implemented by the verifier by
reserving 2 8byte slots in the program stack and then calling
arch_bpf_timed_may_goto() in a loop with the stack offset of these two
slots in BPF_REG_AX. It expects the function to put a timestamp in the
first slot and the returned count in BPF_REG_AX is put into the second
slot by a store instruction emitted by the verifier.

arch_bpf_timed_may_goto() is special as it receives the parameter in
BPF_REG_AX and is expected to return the result in BPF_REG_AX as well.
It can't clobber any caller saved registers because verifier doesn't
save anything before emitting the call.

So, arch_bpf_timed_may_goto() is implemented in assembly so the exact
registers that are stored/restored can be controlled (BPF caller saved
registers here) and it also needs to take care of moving arguments and
return values to and from BPF_REG_AX <-> arm64 R0.

So, arch_bpf_timed_may_goto() acts as a trampoline to call
bpf_check_timed_may_goto() which does the main logic of placing the
timestamp and returning the count.

All tests that use may_goto instruction pass after the changing some of
them in patch 2

 #404     stream_errors:OK
 [...]
 #406/2   stream_success/stream_cond_break:OK
 [...]
 #494/23  verifier_bpf_fastcall/may_goto_interaction_x86_64:SKIP
 #494/24  verifier_bpf_fastcall/may_goto_interaction_arm64:OK
 [...]
 #539/1   verifier_may_goto_1/may_goto 0:OK
 #539/2   verifier_may_goto_1/batch 2 of may_goto 0:OK
 #539/3   verifier_may_goto_1/may_goto batch with offsets 2/1/0:OK
 #539/4   verifier_may_goto_1/may_goto batch with offsets 2/0:OK
 #539     verifier_may_goto_1:OK
 #540/1   verifier_may_goto_2/C code with may_goto 0:OK
 #540     verifier_may_goto_2:OK
 Summary: 7/16 PASSED, 25 SKIPPED, 0 FAILED
====================

Link: https://patch.msgid.link/20250827113245.52629-1-puranjay@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 4c229f33 22b22bf9

arch/arm64/net/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -2,4 +2,4 @@ 
#

# ARM64 networking code

#

obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
 
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o

arch/arm64/net/bpf_jit_comp.c

+12 −1
Original line number Diff line number Diff line
@@ -1558,6 +1558,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, 
		if (ret < 0)

			return ret;

		emit_call(func_addr, ctx);

		/*

		 * Call to arch_bpf_timed_may_goto() is emitted by the

		 * verifier and called with custom calling convention with

		 * first argument and return value in BPF_REG_AX (x9).

		 */

		if (func_addr != (u64)arch_bpf_timed_may_goto)

			emit(A64_MOV(1, r0, A64_R(0)), ctx);

		break;

	}
@@ -3038,6 +3044,11 @@ bool bpf_jit_bypass_spec_v4(void) 
	return true;

}



bool bpf_jit_supports_timed_may_goto(void)

{

	return true;

}



bool bpf_jit_inlines_helper_call(s32 imm)

{

	switch (imm) {

arch/arm64/net/bpf_timed_may_goto.S

0 → 100644
+40 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/* Copyright (c) 2025 Puranjay Mohan <puranjay@kernel.org> */



#include <linux/linkage.h>



SYM_FUNC_START(arch_bpf_timed_may_goto)

	/* Allocate stack space and emit frame record */

	stp     x29, x30, [sp, #-64]!

	mov     x29, sp



	/* Save BPF registers R0 - R5 (x7, x0-x4)*/

	stp	x7, x0, [sp, #16]

	stp	x1, x2, [sp, #32]

	stp	x3, x4, [sp, #48]



	/*

	 * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP

	 * (x25) to get the pointer to count and timestamp and pass it as the

	 * first argument in x0.

	 *

	 * Before generating the call to arch_bpf_timed_may_goto, the verifier

	 * generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP -

	 * stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64

	 * jit in this case.

	 */

	add	x0, x9, x25

	bl	bpf_check_timed_may_goto

	/* BPF_REG_AX(x9) will be stored into count, so move return value to it. */

	mov	x9, x0



	/* Restore BPF registers R0 - R5 (x7, x0-x4) */

	ldp	x7, x0, [sp, #16]

	ldp	x1, x2, [sp, #32]

	ldp	x3, x4, [sp, #48]



	/* Restore FP and LR */

	ldp     x29, x30, [sp], #64



	ret

SYM_FUNC_END(arch_bpf_timed_may_goto)

tools/testing/selftests/bpf/prog_tests/stream.c

+1 −1
Original line number Diff line number Diff line
@@ -77,7 +77,7 @@ void test_stream_errors(void) 
		ASSERT_OK(ret, "ret");

		ASSERT_OK(opts.retval, "retval");



#if !defined(__x86_64__) && !defined(__s390x__)

#if !defined(__x86_64__) && !defined(__s390x__) && !defined(__aarch64__)

		ASSERT_TRUE(1, "Timed may_goto unsupported, skip.");

		if (i == 0) {

			ret = bpf_prog_stream_read(prog_fd, 2, buf, sizeof(buf), &ropts);

tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c

+16 −11
Original line number Diff line number Diff line
@@ -660,19 +660,24 @@ __naked void may_goto_interaction_x86_64(void) 


SEC("raw_tp")

__arch_arm64

__log_level(4) __msg("stack depth 16")

/* may_goto counter at -16 */

__xlated("0: *(u64 *)(r10 -16) =")

__xlated("1: r1 = 1")

__xlated("2: call bpf_get_smp_processor_id")

__log_level(4) __msg("stack depth 24")

/* may_goto counter at -24 */

__xlated("0: *(u64 *)(r10 -24) =")

/* may_goto timestamp at -16 */

__xlated("1: *(u64 *)(r10 -16) =")

__xlated("2: r1 = 1")

__xlated("3: call bpf_get_smp_processor_id")

/* may_goto expansion starts */

__xlated("3: r11 = *(u64 *)(r10 -16)")

__xlated("4: if r11 == 0x0 goto pc+3")

__xlated("5: r11 -= 1")

__xlated("6: *(u64 *)(r10 -16) = r11")

__xlated("4: r11 = *(u64 *)(r10 -24)")

__xlated("5: if r11 == 0x0 goto pc+6")

__xlated("6: r11 -= 1")

__xlated("7: if r11 != 0x0 goto pc+2")

__xlated("8: r11 = -24")

__xlated("9: call unknown")

__xlated("10: *(u64 *)(r10 -24) = r11")

/* may_goto expansion ends */

__xlated("7: *(u64 *)(r10 -8) = r1")

__xlated("8: exit")

__xlated("11: *(u64 *)(r10 -8) = r1")

__xlated("12: exit")

__success

__naked void may_goto_interaction_arm64(void)

{