Commit 2e244a72 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-raw-tracepoint-support-for-bpf-cookie' 

Andrii Nakryiko says:

====================
BPF raw tracepoint support for BPF cookie

Add ability to specify and retrieve BPF cookie for raw tracepoint programs.
Both BTF-aware (SEC("tp_btf")) and non-BTF-aware (SEC("raw_tp")) are
supported, as they are exactly the same at runtime.

This issue recently came up in production use cases, where custom tried to
switch from slower classic tracepoints to raw tracepoints and ran into this
limitation. Luckily, it's not that hard to support this for raw_tp programs.

v2->v3:
  - s/bpf_raw_tp_open/bpf_raw_tracepoint_open_opts/ (Alexei, Eduard);
v1->v2:
  - fixed type definition for stubs of bpf_probe_{register,unregister};
  - added __u32 :u32 and aligned raw_tp fields (Jiri);
  - added Stanislav's ack.
====================

Link: https://lore.kernel.org/r/20240319233852.1977493-1-andrii@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents f803bcf9 51146ff0

include/linux/bpf.h

+6 −0
Original line number Diff line number Diff line
@@ -1607,6 +1607,12 @@ struct bpf_tracing_link { 
	struct bpf_prog *tgt_prog;

};



struct bpf_raw_tp_link {

	struct bpf_link link;

	struct bpf_raw_event_map *btp;

	u64 cookie;

};



struct bpf_link_primer {

	struct bpf_link *link;

	struct file *file;

include/linux/trace_events.h

+20 −16
Original line number Diff line number Diff line
@@ -759,8 +759,11 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx); 
int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);

void perf_event_detach_bpf_prog(struct perf_event *event);

int perf_event_query_prog_array(struct perf_event *event, void __user *info);

int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog);

int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog);



struct bpf_raw_tp_link;

int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);

int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);



struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name);

void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);

int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
@@ -788,11 +791,12 @@ perf_event_query_prog_array(struct perf_event *event, void __user *info) 
{

	return -EOPNOTSUPP;

}

static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *p)

struct bpf_raw_tp_link;

static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)

{

	return -EOPNOTSUPP;

}

static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *p)

static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)

{

	return -EOPNOTSUPP;

}
@@ -903,31 +907,31 @@ void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp); 
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);

void perf_event_free_bpf_prog(struct perf_event *event);



void bpf_trace_run1(struct bpf_prog *prog, u64 arg1);

void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2);

void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run1(struct bpf_raw_tp_link *link, u64 arg1);

void bpf_trace_run2(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2);

void bpf_trace_run3(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3);

void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run4(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3, u64 arg4);

void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run5(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3, u64 arg4, u64 arg5);

void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run6(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3, u64 arg4, u64 arg5, u64 arg6);

void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run7(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);

void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run8(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,

		    u64 arg8);

void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run9(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,

		    u64 arg8, u64 arg9);

void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run10(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,

		     u64 arg8, u64 arg9, u64 arg10);

void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run11(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,

		     u64 arg8, u64 arg9, u64 arg10, u64 arg11);

void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2,

void bpf_trace_run12(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,

		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,

		     u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);

void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,

include/trace/bpf_probe.h

+1 −2
Original line number Diff line number Diff line
@@ -46,8 +46,7 @@ 
static notrace void							\

__bpf_trace_##call(void *__data, proto)					\

{									\

	struct bpf_prog *prog = __data;					\

	CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args));	\

	CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(__data, CAST_TO_U64(args));	\

}



#undef DECLARE_EVENT_CLASS

include/uapi/linux/bpf.h

+4 −2
Original line number Diff line number Diff line
@@ -1664,6 +1664,8 @@ union bpf_attr { 
	struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */

		__u64		name;

		__u32		prog_fd;

		__u32		:32;

		__aligned_u64	cookie;

	} raw_tracepoint;



	struct { /* anonymous struct for BPF_BTF_LOAD */

kernel/bpf/syscall.c

+11 −11
Original line number Diff line number Diff line
@@ -3469,17 +3469,12 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, 
	return err;

}



struct bpf_raw_tp_link {

	struct bpf_link link;

	struct bpf_raw_event_map *btp;

};



static void bpf_raw_tp_link_release(struct bpf_link *link)

{

	struct bpf_raw_tp_link *raw_tp =

		container_of(link, struct bpf_raw_tp_link, link);



	bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);

	bpf_probe_unregister(raw_tp->btp, raw_tp);

	bpf_put_raw_tracepoint(raw_tp->btp);

}
@@ -3779,7 +3774,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro 
#endif /* CONFIG_PERF_EVENTS */



static int bpf_raw_tp_link_attach(struct bpf_prog *prog,

				  const char __user *user_tp_name)

				  const char __user *user_tp_name, u64 cookie)

{

	struct bpf_link_primer link_primer;

	struct bpf_raw_tp_link *link;
@@ -3826,6 +3821,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, 
	bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,

		      &bpf_raw_tp_link_lops, prog);

	link->btp = btp;

	link->cookie = cookie;



	err = bpf_link_prime(&link->link, &link_primer);

	if (err) {
@@ -3833,7 +3829,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, 
		goto out_put_btp;

	}



	err = bpf_probe_register(link->btp, prog);

	err = bpf_probe_register(link->btp, link);

	if (err) {

		bpf_link_cleanup(&link_primer);

		goto out_put_btp;
@@ -3846,11 +3842,13 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, 
	return err;

}



#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd

#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookie



static int bpf_raw_tracepoint_open(const union bpf_attr *attr)

{

	struct bpf_prog *prog;

	void __user *tp_name;

	__u64 cookie;

	int fd;



	if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
@@ -3860,7 +3858,9 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 
	if (IS_ERR(prog))

		return PTR_ERR(prog);



	fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name));

	tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);

	cookie = attr->raw_tracepoint.cookie;

	fd = bpf_raw_tp_link_attach(prog, tp_name, cookie);

	if (fd < 0)

		bpf_prog_put(prog);

	return fd;
@@ -5198,7 +5198,7 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) 
			goto out;

		}

		if (prog->expected_attach_type == BPF_TRACE_RAW_TP)

			ret = bpf_raw_tp_link_attach(prog, NULL);

			ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie);

		else if (prog->expected_attach_type == BPF_TRACE_ITER)

			ret = bpf_iter_link_attach(attr, uattr, prog);

		else if (prog->expected_attach_type == BPF_LSM_CGROUP)