Commit 0e73ef1d authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'bpf: Add missed stats for kprobes' 

Jiri Olsa says:

====================
hi,
at the moment we can't retrieve the number of missed kprobe
executions and subsequent execution of BPF programs.

This patchset adds:
  - counting of missed execution on attach layer for:
    . kprobes attached through perf link (kprobe/ftrace)
    . kprobes attached through kprobe.multi link (fprobe)
  - counting of recursion_misses for BPF kprobe programs

It's still technically possible to create kprobe without perf link (using
SET_BPF perf ioctl) in which case we don't have a way to retrieve the kprobe's
'missed' count. However both libbpf and cilium/ebpf libraries use perf link
if it's available, and for old kernels without perf link support we can use
BPF program to retrieve the kprobe missed count.

v3 changes:
  - added acks [Song]
  - make test_missed not serial [Andrii]

Also available at:
  https://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git


  bpf/missed_stats

thanks,
jirka
====================

Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents e0fa6523 85981e0f

include/linux/bpf.h

+16 −0
Original line number Diff line number Diff line
@@ -2922,6 +2922,22 @@ static inline int sock_map_bpf_prog_query(const union bpf_attr *attr, 
#endif /* CONFIG_BPF_SYSCALL */

#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */



static __always_inline void

bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)

{

	const struct bpf_prog_array_item *item;

	struct bpf_prog *prog;



	if (unlikely(!array))

		return;



	item = &array->items[0];

	while ((prog = READ_ONCE(item->prog))) {

		bpf_prog_inc_misses_counter(prog);

		item++;

	}

}



#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)

void bpf_sk_reuseport_detach(struct sock *sk);

int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,

include/linux/trace_events.h

+4 −2
Original line number Diff line number Diff line
@@ -761,7 +761,8 @@ struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name); 
void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);

int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,

			    u32 *fd_type, const char **buf,

			    u64 *probe_offset, u64 *probe_addr);

			    u64 *probe_offset, u64 *probe_addr,

			    unsigned long *missed);

int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);

int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);

#else
@@ -801,7 +802,7 @@ static inline void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 
static inline int bpf_get_perf_event_info(const struct perf_event *event,

					  u32 *prog_id, u32 *fd_type,

					  const char **buf, u64 *probe_offset,

					  u64 *probe_addr)

					  u64 *probe_addr, unsigned long *missed)

{

	return -EOPNOTSUPP;

}
@@ -877,6 +878,7 @@ extern void perf_kprobe_destroy(struct perf_event *event); 
extern int bpf_get_kprobe_info(const struct perf_event *event,

			       u32 *fd_type, const char **symbol,

			       u64 *probe_offset, u64 *probe_addr,

			       unsigned long *missed,

			       bool perf_type_tracepoint);

#endif

#ifdef CONFIG_UPROBE_EVENTS

include/uapi/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -6532,6 +6532,7 @@ struct bpf_link_info { 
			__aligned_u64 addrs;

			__u32 count; /* in/out: kprobe_multi function count */

			__u32 flags;

			__u64 missed;

		} kprobe_multi;

		struct {

			__u32 type; /* enum bpf_perf_event_type */
@@ -6547,6 +6548,7 @@ struct bpf_link_info { 
					__u32 name_len;

					__u32 offset; /* offset from func_name */

					__u64 addr;

					__u64 missed;

				} kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */

				struct {

					__aligned_u64 tp_name;   /* in/out */

kernel/bpf/syscall.c

+8 −6
Original line number Diff line number Diff line
@@ -3374,7 +3374,7 @@ static void bpf_perf_link_dealloc(struct bpf_link *link) 
static int bpf_perf_link_fill_common(const struct perf_event *event,

				     char __user *uname, u32 ulen,

				     u64 *probe_offset, u64 *probe_addr,

				     u32 *fd_type)

				     u32 *fd_type, unsigned long *missed)

{

	const char *buf;

	u32 prog_id;
@@ -3385,7 +3385,7 @@ static int bpf_perf_link_fill_common(const struct perf_event *event, 
		return -EINVAL;



	err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,

				      probe_offset, probe_addr);

				      probe_offset, probe_addr, missed);

	if (err)

		return err;

	if (!uname)
@@ -3408,6 +3408,7 @@ static int bpf_perf_link_fill_common(const struct perf_event *event, 
static int bpf_perf_link_fill_kprobe(const struct perf_event *event,

				     struct bpf_link_info *info)

{

	unsigned long missed;

	char __user *uname;

	u64 addr, offset;

	u32 ulen, type;
@@ -3416,7 +3417,7 @@ static int bpf_perf_link_fill_kprobe(const struct perf_event *event, 
	uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);

	ulen = info->perf_event.kprobe.name_len;

	err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,

					&type);

					&type, &missed);

	if (err)

		return err;

	if (type == BPF_FD_TYPE_KRETPROBE)
@@ -3425,6 +3426,7 @@ static int bpf_perf_link_fill_kprobe(const struct perf_event *event, 
		info->perf_event.type = BPF_PERF_EVENT_KPROBE;



	info->perf_event.kprobe.offset = offset;

	info->perf_event.kprobe.missed = missed;

	if (!kallsyms_show_value(current_cred()))

		addr = 0;

	info->perf_event.kprobe.addr = addr;
@@ -3444,7 +3446,7 @@ static int bpf_perf_link_fill_uprobe(const struct perf_event *event, 
	uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);

	ulen = info->perf_event.uprobe.name_len;

	err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,

					&type);

					&type, NULL);

	if (err)

		return err;
@@ -3480,7 +3482,7 @@ static int bpf_perf_link_fill_tracepoint(const struct perf_event *event, 
	uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);

	ulen = info->perf_event.tracepoint.name_len;

	info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;

	return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL);

	return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL, NULL);

}



static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
@@ -4813,7 +4815,7 @@ static int bpf_task_fd_query(const union bpf_attr *attr, 


		err = bpf_get_perf_event_info(event, &prog_id, &fd_type,

					      &buf, &probe_offset,

					      &probe_addr);

					      &probe_addr, NULL);

		if (!err)

			err = bpf_task_fd_query_copy(attr, uattr, prog_id,

						     fd_type, buf,

kernel/trace/bpf_trace.c

+8 −2
Original line number Diff line number Diff line
@@ -117,6 +117,9 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 
		 * and don't send kprobe event into ring-buffer,

		 * so return zero here

		 */

		rcu_read_lock();

		bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array));

		rcu_read_unlock();

		ret = 0;

		goto out;

	}
@@ -2384,7 +2387,8 @@ int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 


int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,

			    u32 *fd_type, const char **buf,

			    u64 *probe_offset, u64 *probe_addr)

			    u64 *probe_offset, u64 *probe_addr,

			    unsigned long *missed)

{

	bool is_tracepoint, is_syscall_tp;

	struct bpf_prog *prog;
@@ -2419,7 +2423,7 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 
#ifdef CONFIG_KPROBE_EVENTS

		if (flags & TRACE_EVENT_FL_KPROBE)

			err = bpf_get_kprobe_info(event, fd_type, buf,

						  probe_offset, probe_addr,

						  probe_offset, probe_addr, missed,

						  event->attr.type == PERF_TYPE_TRACEPOINT);

#endif

#ifdef CONFIG_UPROBE_EVENTS
@@ -2614,6 +2618,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, 
	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);

	info->kprobe_multi.count = kmulti_link->cnt;

	info->kprobe_multi.flags = kmulti_link->flags;

	info->kprobe_multi.missed = kmulti_link->fp.nmissed;



	if (!uaddrs)

		return 0;
@@ -2710,6 +2715,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, 
	int err;



	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {

		bpf_prog_inc_misses_counter(link->link.prog);

		err = 0;

		goto out;

	}