Commit ba335bf3 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-x86-inline-bpf_get_current_task-for-x86_64' 

Menglong Dong says:

====================
bpf, x86: inline bpf_get_current_task() for x86_64

Inline bpf_get_current_task() and bpf_get_current_task_btf() for x86_64
to obtain better performance, and add the testcase for it.

Changes since v5:
* remove unnecessary 'ifdef' and __description in the selftests
* v5: https://lore.kernel.org/bpf/20260119070246.249499-1-dongml2@chinatelecom.cn/

Changes since v4:
* don't support the !CONFIG_SMP case
* v4: https://lore.kernel.org/bpf/20260112104529.224645-1-dongml2@chinatelecom.cn/

Changes since v3:
* handle the !CONFIG_SMP case
* ignore the !CONFIG_SMP case in the testcase, as we enable CONFIG_SMP
  for x86_64 in the selftests

Changes since v2:
* implement it in the verifier with BPF_MOV64_PERCPU_REG() instead of in
  x86_64 JIT (Alexei).

Changes since v1:
* add the testcase
* remove the usage of const_current_task
====================

Link: https://patch.msgid.link/20260120070555.233486-1-dongml2@chinatelecom.cn


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 83c9030c 4fca9509

kernel/bpf/verifier.c

+22 −0
Original line number Diff line number Diff line
@@ -18130,6 +18130,10 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) 
	switch (imm) {

#ifdef CONFIG_X86_64

	case BPF_FUNC_get_smp_processor_id:

#ifdef CONFIG_SMP

	case BPF_FUNC_get_current_task_btf:

	case BPF_FUNC_get_current_task:

#endif

		return env->prog->jit_requested && bpf_jit_supports_percpu_insn();

#endif

	default:
@@ -23715,6 +23719,24 @@ static int do_misc_fixups(struct bpf_verifier_env *env) 
			insn      = new_prog->insnsi + i + delta;

			goto next_insn;

		}









		/* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */









		if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) &&









		    verifier_inlines_helper_call(env, insn->imm)) {









			insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&current_task);









			insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);









			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);









			cnt = 3;

















			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);









			if (!new_prog)









				return -ENOMEM;

















			delta    += cnt - 1;









			env->prog = prog = new_prog;









			insn      = new_prog->insnsi + i + delta;









			goto next_insn;









		}









#endif









		/* Implement bpf_get_func_arg inline. */









		if (prog_type == BPF_PROG_TYPE_TRACING &&

























tools/testing/selftests/bpf/prog_tests/verifier.c



+2
−0






























Original line number
Diff line number
Diff line







@@ -112,6 +112,7 @@







#include "verifier_xdp_direct_packet_access.skel.h"









#include "verifier_bits_iter.skel.h"









#include "verifier_lsm.skel.h"









#include "verifier_jit_inline.skel.h"









#include "irq.skel.h"



















#define MAX_ENTRIES 11








@@ -255,6 +256,7 @@ void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); }







void test_verifier_lsm(void)                  { RUN(verifier_lsm); }









void test_irq(void)			      { RUN(irq); }









void test_verifier_mtu(void)		      { RUN(verifier_mtu); }









void test_verifier_jit_inline(void)               { RUN(verifier_jit_inline); }



















static int init_test_val_map(struct bpf_object *obj, char *map_name)









{
































tools/testing/selftests/bpf/progs/verifier_jit_inline.c
0 → 100644



+20
−0






























Original line number
Diff line number
Diff line









// SPDX-License-Identifier: GPL-2.0



















#include <vmlinux.h>









#include <bpf/bpf_helpers.h>









#include "bpf_misc.h"



















SEC("fentry/bpf_fentry_test1")









__success __retval(0)









__arch_x86_64









__jited("	addq	%gs:{{.*}}, %rax")









__arch_arm64









__jited("	mrs	x7, SP_EL0")









int inline_bpf_get_current_task(void)









{









	bpf_get_current_task();



















	return 0;









}



















char _license[] SEC("license") = "GPL";