Commit 06ebfd11 authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'bpf-introduce-kprobe_multi-session-attach' 

Jiri Olsa says:

====================
bpf: Introduce kprobe_multi session attach

hi,
adding support to attach kprobe program through kprobe_multi link
in a session mode, which means:
  - program is attached to both function entry and return
  - entry program can decided if the return program gets executed
  - entry program can share u64 cookie value with return program

The initial RFC for this was posted in [0] and later discussed more
and which ended up with the session idea [1]

Having entry together with return probe for given function is common
use case for tetragon, bpftrace and most likely for others.

At the moment if we want both entry and return probe to execute bpf
program we need to create two (entry and return probe) links. The link
for return probe creates extra entry probe to setup the return probe.
The extra entry probe execution could be omitted if we had a way to
use just single link for both entry and exit probe.

In addition the possibility to control the return program execution
and sharing data within entry and return probe allows for other use
cases.

v2 changes:
  - renamed BPF_TRACE_KPROBE_MULTI_SESSION to BPF_TRACE_KPROBE_SESSION
    [Andrii]
  - use arrays for results in selftest [Andrii]
  - various small selftests and libbpf changes [Andrii]
  - moved the verifier cookie setup earlier in check_kfunc_call [Andrii]
  - added acks

Also available at:
  https://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git
  bpf/session_data

thanks,
jirka

[0] https://lore.kernel.org/bpf/20240207153550.856536-1-jolsa@kernel.org/
[1] https://lore.kernel.org/bpf/20240228090242.4040210-1-jolsa@kernel.org/
---
====================

Link: https://lore.kernel.org/r/20240430112830.1184228-1-jolsa@kernel.org


Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents 05cbc217 a3a51133

include/uapi/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -1115,6 +1115,7 @@ enum bpf_attach_type { 
	BPF_CGROUP_UNIX_GETSOCKNAME,

	BPF_NETKIT_PRIMARY,

	BPF_NETKIT_PEER,

	BPF_TRACE_KPROBE_SESSION,

	__MAX_BPF_ATTACH_TYPE

};

kernel/bpf/btf.c

+3 −0
Original line number Diff line number Diff line
@@ -218,6 +218,7 @@ enum btf_kfunc_hook { 
	BTF_KFUNC_HOOK_SOCKET_FILTER,

	BTF_KFUNC_HOOK_LWT,

	BTF_KFUNC_HOOK_NETFILTER,

	BTF_KFUNC_HOOK_KPROBE,

	BTF_KFUNC_HOOK_MAX,

};
@@ -8157,6 +8158,8 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) 
		return BTF_KFUNC_HOOK_LWT;

	case BPF_PROG_TYPE_NETFILTER:

		return BTF_KFUNC_HOOK_NETFILTER;

	case BPF_PROG_TYPE_KPROBE:

		return BTF_KFUNC_HOOK_KPROBE;

	default:

		return BTF_KFUNC_HOOK_MAX;

	}

kernel/bpf/syscall.c

+6 −1
Original line number Diff line number Diff line
@@ -4016,11 +4016,15 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 
		if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI &&

		    attach_type != BPF_TRACE_KPROBE_MULTI)

			return -EINVAL;

		if (prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION &&

		    attach_type != BPF_TRACE_KPROBE_SESSION)

			return -EINVAL;

		if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&

		    attach_type != BPF_TRACE_UPROBE_MULTI)

			return -EINVAL;

		if (attach_type != BPF_PERF_EVENT &&

		    attach_type != BPF_TRACE_KPROBE_MULTI &&

		    attach_type != BPF_TRACE_KPROBE_SESSION &&

		    attach_type != BPF_TRACE_UPROBE_MULTI)

			return -EINVAL;

		return 0;
@@ -5281,7 +5285,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) 
	case BPF_PROG_TYPE_KPROBE:

		if (attr->link_create.attach_type == BPF_PERF_EVENT)

			ret = bpf_perf_link_attach(attr, prog);

		else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI)

		else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||

			 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)

			ret = bpf_kprobe_multi_link_attach(attr, prog);

		else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)

			ret = bpf_uprobe_multi_link_attach(attr, prog);

kernel/bpf/verifier.c

+7 −0
Original line number Diff line number Diff line
@@ -11063,6 +11063,7 @@ enum special_kfunc_type { 
	KF_bpf_preempt_disable,

	KF_bpf_preempt_enable,

	KF_bpf_iter_css_task_new,

	KF_bpf_session_cookie,

};









BTF_SET_START(special_kfunc_set)








@@ -11123,6 +11124,7 @@ BTF_ID(func, bpf_iter_css_task_new)







#else









BTF_ID_UNUSED









#endif









BTF_ID(func, bpf_session_cookie)

















static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)









{








@@ -12294,6 +12296,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,







		}









	}

















	if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) {









		meta.r0_size = sizeof(u64);









		meta.r0_rdonly = false;









	}

















	if (is_bpf_wq_set_callback_impl_kfunc(meta.func_id)) {









		err = push_callback_call(env, insn, insn_idx, meta.subprogno,









					 set_timer_callback_state);
































kernel/trace/bpf_trace.c



+92
−14






























Original line number
Diff line number
Diff line







@@ -1631,6 +1631,17 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)







	}









}



















static bool is_kprobe_multi(const struct bpf_prog *prog)









{









	return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ||









	       prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;









}



















static inline bool is_kprobe_session(const struct bpf_prog *prog)









{









	return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;









}



















static const struct bpf_func_proto *









kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)









{








@@ -1646,13 +1657,13 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)







		return &bpf_override_return_proto;









#endif









	case BPF_FUNC_get_func_ip:









		if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI)









		if (is_kprobe_multi(prog))









			return &bpf_get_func_ip_proto_kprobe_multi;









		if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)









			return &bpf_get_func_ip_proto_uprobe_multi;









		return &bpf_get_func_ip_proto_kprobe;









	case BPF_FUNC_get_attach_cookie:









		if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI)









		if (is_kprobe_multi(prog))









			return &bpf_get_attach_cookie_proto_kmulti;









		if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)









			return &bpf_get_attach_cookie_proto_umulti;








@@ -2585,6 +2596,12 @@ static int __init bpf_event_init(void)







fs_initcall(bpf_event_init);









#endif /* CONFIG_MODULES */



















struct bpf_session_run_ctx {









	struct bpf_run_ctx run_ctx;









	bool is_return;









	void *data;









};



















#ifdef CONFIG_FPROBE









struct bpf_kprobe_multi_link {









	struct bpf_link link;








@@ -2598,7 +2615,7 @@ struct bpf_kprobe_multi_link {







};



















struct bpf_kprobe_multi_run_ctx {









	struct bpf_run_ctx run_ctx;









	struct bpf_session_run_ctx session_ctx;









	struct bpf_kprobe_multi_link *link;









	unsigned long entry_ip;









};








@@ -2777,7 +2794,8 @@ static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx)

















	if (WARN_ON_ONCE(!ctx))









		return 0;









	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, run_ctx);









	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx,









			       session_ctx.run_ctx);









	link = run_ctx->link;









	if (!link->cookies)









		return 0;








@@ -2794,15 +2812,21 @@ static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)







{









	struct bpf_kprobe_multi_run_ctx *run_ctx;



















	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, run_ctx);









	run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx,









			       session_ctx.run_ctx);









	return run_ctx->entry_ip;









}



















static int









kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,









			   unsigned long entry_ip, struct pt_regs *regs)









			   unsigned long entry_ip, struct pt_regs *regs,









			   bool is_return, void *data)









{









	struct bpf_kprobe_multi_run_ctx run_ctx = {









		.session_ctx = {









			.is_return = is_return,









			.data = data,









		},









		.link = link,









		.entry_ip = entry_ip,









	};








@@ -2817,7 +2841,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,

















	migrate_disable();









	rcu_read_lock();









	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);









	old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);









	err = bpf_prog_run(link->link.prog, regs);









	bpf_reset_run_ctx(old_run_ctx);









	rcu_read_unlock();








@@ -2834,10 +2858,11 @@ kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip,







			  void *data)









{









	struct bpf_kprobe_multi_link *link;









	int err;



















	link = container_of(fp, struct bpf_kprobe_multi_link, fp);









	kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs);









	return 0;









	err = kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs, false, data);









	return is_kprobe_session(link->link.prog) ? err : 0;









}



















static void








@@ -2848,7 +2873,7 @@ kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip,







	struct bpf_kprobe_multi_link *link;



















	link = container_of(fp, struct bpf_kprobe_multi_link, fp);









	kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs);









	kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs, true, data);









}



















static int symbols_cmp_r(const void *a, const void *b, const void *priv)








@@ -2981,7 +3006,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr







	if (sizeof(u64) != sizeof(void *))









		return -EOPNOTSUPP;



















	if (prog->expected_attach_type != BPF_TRACE_KPROBE_MULTI)









	if (!is_kprobe_multi(prog))









		return -EINVAL;



















	flags = attr->link_create.kprobe_multi.flags;








@@ -3062,10 +3087,12 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr







	if (err)









		goto error;



















	if (flags & BPF_F_KPROBE_MULTI_RETURN)









		link->fp.exit_handler = kprobe_multi_link_exit_handler;









	else









	if (!(flags & BPF_F_KPROBE_MULTI_RETURN))









		link->fp.entry_handler = kprobe_multi_link_handler;









	if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog))









		link->fp.exit_handler = kprobe_multi_link_exit_handler;









	if (is_kprobe_session(prog))









		link->fp.entry_data_size = sizeof(u64);



















	link->addrs = addrs;









	link->cookies = cookies;








@@ -3491,3 +3518,54 @@ static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)







	return 0;









}









#endif /* CONFIG_UPROBES */



















#ifdef CONFIG_FPROBE









__bpf_kfunc_start_defs();



















__bpf_kfunc bool bpf_session_is_return(void)









{









	struct bpf_session_run_ctx *session_ctx;



















	session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx);









	return session_ctx->is_return;









}



















__bpf_kfunc __u64 *bpf_session_cookie(void)









{









	struct bpf_session_run_ctx *session_ctx;



















	session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx);









	return session_ctx->data;









}



















__bpf_kfunc_end_defs();



















BTF_KFUNCS_START(kprobe_multi_kfunc_set_ids)









BTF_ID_FLAGS(func, bpf_session_is_return)









BTF_ID_FLAGS(func, bpf_session_cookie)









BTF_KFUNCS_END(kprobe_multi_kfunc_set_ids)



















static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id)









{









	if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))









		return 0;



















	if (!is_kprobe_session(prog))









		return -EACCES;



















	return 0;









}



















static const struct btf_kfunc_id_set bpf_kprobe_multi_kfunc_set = {









	.owner = THIS_MODULE,









	.set = &kprobe_multi_kfunc_set_ids,









	.filter = bpf_kprobe_multi_filter,









};



















static int __init bpf_kprobe_multi_kfuncs_init(void)









{









	return register_btf_kfunc_id_set(BPF_PROG_TYPE_KPROBE, &bpf_kprobe_multi_kfunc_set);









}



















late_initcall(bpf_kprobe_multi_kfuncs_init);









#endif