Commit 9a675ba5 authored by Sebastian Andrzej Siewior's avatar Sebastian Andrzej Siewior Committed by Daniel Borkmann
Browse files

net, bpf: Add a warning if NAPI cb missed xdp_do_flush(). 



A few drivers were missing a xdp_do_flush() invocation after
XDP_REDIRECT.

Add three helper functions each for one of the per-CPU lists. Return
true if the per-CPU list is non-empty and flush the list.

Add xdp_do_check_flushed() which invokes each helper functions and
creates a warning if one of the functions had a non-empty list.

Hide everything behind CONFIG_DEBUG_NET.

Suggested-by: default avatarJesper Dangaard Brouer <hawk@kernel.org>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Reviewed-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Acked-by: default avatarJakub Kicinski <kuba@kernel.org>
Acked-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Link: https://lore.kernel.org/bpf/20231016125738.Yt79p1uF@linutronix.de
parent 137df118

include/linux/bpf.h

+3 −0
Original line number Diff line number Diff line
@@ -2478,6 +2478,9 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, 
		     enum bpf_dynptr_type type, u32 offset, u32 size);

void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);

void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);



bool dev_check_flush(void);

bool cpu_map_check_flush(void);

#else /* !CONFIG_BPF_SYSCALL */

static inline struct bpf_prog *bpf_prog_get(u32 ufd)

{

include/net/xdp_sock.h

+9 −0
Original line number Diff line number Diff line
@@ -109,4 +109,13 @@ static inline void __xsk_map_flush(void) 


#endif /* CONFIG_XDP_SOCKETS */



#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)

bool xsk_map_check_flush(void);

#else

static inline bool xsk_map_check_flush(void)

{

	return false;

}

#endif



#endif /* _LINUX_XDP_SOCK_H */

kernel/bpf/cpumap.c

+10 −0
Original line number Diff line number Diff line
@@ -764,6 +764,16 @@ void __cpu_map_flush(void) 
	}

}



#ifdef CONFIG_DEBUG_NET

bool cpu_map_check_flush(void)

{

	if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))

		return false;

	__cpu_map_flush();

	return true;

}

#endif



static int __init cpu_map_init(void)

{

	int cpu;

kernel/bpf/devmap.c

+10 −0
Original line number Diff line number Diff line
@@ -418,6 +418,16 @@ void __dev_flush(void) 
	}

}



#ifdef CONFIG_DEBUG_NET

bool dev_check_flush(void)

{

	if (list_empty(this_cpu_ptr(&dev_flush_list)))

		return false;

	__dev_flush();

	return true;

}

#endif



/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or

 * by local_bh_disable() (from XDP calls inside NAPI). The

 * rcu_read_lock_bh_held() below makes lockdep accept both.

net/core/dev.c

+2 −0
Original line number Diff line number Diff line
@@ -6535,6 +6535,8 @@ static int __napi_poll(struct napi_struct *n, bool *repoll) 
	if (test_bit(NAPI_STATE_SCHED, &n->state)) {

		work = n->poll(n, weight);

		trace_napi_poll(n, work, weight);



		xdp_do_check_flushed(n);

	}



	if (unlikely(work > weight))