Commit 66c84731 authored by Andrii Nakryiko's avatar Andrii Nakryiko Committed by Alexei Starovoitov
Browse files

bpf: move sleepable flag from bpf_prog_aux to bpf_prog 



prog->aux->sleepable is checked very frequently as part of (some) BPF
program run hot paths. So this extra aux indirection seems wasteful and
on busy systems might cause unnecessary memory cache misses.

Let's move sleepable flag into prog itself to eliminate unnecessary
pointer dereference.

Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Acked-by: default avatarJiri Olsa <jolsa@kernel.org>
Message-ID: <20240309004739.2961431-1-andrii@kernel.org>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent d6170e4a

include/linux/bpf.h

+4 −4
Original line number Diff line number Diff line
@@ -1455,7 +1455,6 @@ struct bpf_prog_aux { 
	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */

	bool attach_tracing_prog; /* true if tracing another tracing program */

	bool func_proto_unreliable;

	bool sleepable;

	bool tail_call_reachable;

	bool xdp_has_frags;

	bool exception_cb;
@@ -1541,7 +1540,8 @@ struct bpf_prog { 
				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */

				call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */

				call_get_func_ip:1, /* Do we call get_func_ip() */

				tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */

				tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */

				sleepable:1;	/* BPF program is sleepable */

	enum bpf_prog_type	type;		/* Type of BPF program */

	enum bpf_attach_type	expected_attach_type; /* For some prog types */

	u32			len;		/* Number of filter blocks */
@@ -2112,14 +2112,14 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu, 
	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);

	item = &array->items[0];

	while ((prog = READ_ONCE(item->prog))) {

		if (!prog->aux->sleepable)

		if (!prog->sleepable)

			rcu_read_lock();



		run_ctx.bpf_cookie = item->bpf_cookie;

		ret &= run_prog(prog, ctx);

		item++;



		if (!prog->aux->sleepable)

		if (!prog->sleepable)

			rcu_read_unlock();

	}

	bpf_reset_run_ctx(old_run_ctx);

kernel/bpf/bpf_iter.c

+2 −2
Original line number Diff line number Diff line
@@ -548,7 +548,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, 
		return -ENOENT;



	/* Only allow sleepable program for resched-able iterator */

	if (prog->aux->sleepable && !bpf_iter_target_support_resched(tinfo))

	if (prog->sleepable && !bpf_iter_target_support_resched(tinfo))

		return -EINVAL;



	link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
@@ -697,7 +697,7 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) 
	struct bpf_run_ctx run_ctx, *old_run_ctx;

	int ret;



	if (prog->aux->sleepable) {

	if (prog->sleepable) {

		rcu_read_lock_trace();

		migrate_disable();

		might_fault();

kernel/bpf/core.c

+1 −1
Original line number Diff line number Diff line
@@ -2706,7 +2706,7 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, 
	bool sleepable;

	u32 i;



	sleepable = aux->sleepable;

	sleepable = aux->prog->sleepable;

	for (i = 0; i < len; i++) {

		map = used_maps[i];

		if (map->ops->map_poke_untrack)

kernel/bpf/syscall.c

+3 −3
Original line number Diff line number Diff line
@@ -2248,7 +2248,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) 
		btf_put(prog->aux->attach_btf);



	if (deferred) {

		if (prog->aux->sleepable)

		if (prog->sleepable)

			call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);

		else

			call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
@@ -2813,11 +2813,11 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) 
	}



	prog->expected_attach_type = attr->expected_attach_type;

	prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE);

	prog->aux->attach_btf = attach_btf;

	prog->aux->attach_btf_id = attr->attach_btf_id;

	prog->aux->dst_prog = dst_prog;

	prog->aux->dev_bound = !!attr->prog_ifindex;

	prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;

	prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;



	/* move token into prog->aux, reuse taken refcnt */
@@ -5554,7 +5554,7 @@ static int bpf_prog_bind_map(union bpf_attr *attr) 
	/* The bpf program will not access the bpf map, but for the sake of

	 * simplicity, increase sleepable_refcnt for sleepable program as well.

	 */

	if (prog->aux->sleepable)

	if (prog->sleepable)

		atomic64_inc(&map->sleepable_refcnt);

	memcpy(used_maps_new, used_maps_old,

	       sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);

kernel/bpf/trampoline.c

+2 −2
Original line number Diff line number Diff line
@@ -1014,7 +1014,7 @@ void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr) 


bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)

{

	bool sleepable = prog->aux->sleepable;

	bool sleepable = prog->sleepable;



	if (bpf_prog_check_recur(prog))

		return sleepable ? __bpf_prog_enter_sleepable_recur :
@@ -1029,7 +1029,7 @@ bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog) 


bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)

{

	bool sleepable = prog->aux->sleepable;

	bool sleepable = prog->sleepable;



	if (bpf_prog_check_recur(prog))

		return sleepable ? __bpf_prog_exit_sleepable_recur :