Commit 401cb7da authored by Sebastian Andrzej Siewior's avatar Sebastian Andrzej Siewior Committed by Jakub Kicinski
Browse files

net: Reference bpf_redirect_info via task_struct on PREEMPT_RT. 



The XDP redirect process is two staged:
- bpf_prog_run_xdp() is invoked to run a eBPF program which inspects the
  packet and makes decisions. While doing that, the per-CPU variable
  bpf_redirect_info is used.

- Afterwards xdp_do_redirect() is invoked and accesses bpf_redirect_info
  and it may also access other per-CPU variables like xskmap_flush_list.

At the very end of the NAPI callback, xdp_do_flush() is invoked which
does not access bpf_redirect_info but will touch the individual per-CPU
lists.

The per-CPU variables are only used in the NAPI callback hence disabling
bottom halves is the only protection mechanism. Users from preemptible
context (like cpu_map_kthread_run()) explicitly disable bottom halves
for protections reasons.
Without locking in local_bh_disable() on PREEMPT_RT this data structure
requires explicit locking.

PREEMPT_RT has forced-threaded interrupts enabled and every
NAPI-callback runs in a thread. If each thread has its own data
structure then locking can be avoided.

Create a struct bpf_net_context which contains struct bpf_redirect_info.
Define the variable on stack, use bpf_net_ctx_set() to save a pointer to
it, bpf_net_ctx_clear() removes it again.
The bpf_net_ctx_set() may nest. For instance a function can be used from
within NET_RX_SOFTIRQ/ net_rx_action which uses bpf_net_ctx_set() and
NET_TX_SOFTIRQ which does not. Therefore only the first invocations
updates the pointer.
Use bpf_net_ctx_get_ri() as a wrapper to retrieve the current struct
bpf_redirect_info. The returned data structure is zero initialized to
ensure nothing is leaked from stack. This is done on first usage of the
struct. bpf_net_ctx_set() sets bpf_redirect_info::kern_flags to 0 to
note that initialisation is required. First invocation of
bpf_net_ctx_get_ri() will memset() the data structure and update
bpf_redirect_info::kern_flags.
bpf_redirect_info::nh is excluded from memset because it is only used
once BPF_F_NEIGH is set which also sets the nh member. The kern_flags is
moved past nh to exclude it from memset.

The pointer to bpf_net_context is saved task's task_struct. Using
always the bpf_net_context approach has the advantage that there is
almost zero differences between PREEMPT_RT and non-PREEMPT_RT builds.

Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Eduard Zingerman <eddyz87@gmail.com>
Cc: Hao Luo <haoluo@google.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: John Fastabend <john.fastabend@gmail.com>
Cc: KP Singh <kpsingh@kernel.org>
Cc: Martin KaFai Lau <martin.lau@linux.dev>
Cc: Song Liu <song@kernel.org>
Cc: Stanislav Fomichev <sdf@google.com>
Cc: Yonghong Song <yonghong.song@linux.dev>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarJesper Dangaard Brouer <hawk@kernel.org>
Reviewed-by: default avatarToke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: https://patch.msgid.link/20240620132727.660738-15-bigeasy@linutronix.de


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 78f520b7

include/linux/filter.h

+46 −10
Original line number Diff line number Diff line
@@ -733,21 +733,59 @@ struct bpf_nh_params { 
	};

};



/* flags for bpf_redirect_info kern_flags */

#define BPF_RI_F_RF_NO_DIRECT	BIT(0)	/* no napi_direct on return_frame */

#define BPF_RI_F_RI_INIT	BIT(1)



struct bpf_redirect_info {

	u64 tgt_index;

	void *tgt_value;

	struct bpf_map *map;

	u32 flags;

	u32 kern_flags;

	u32 map_id;

	enum bpf_map_type map_type;

	struct bpf_nh_params nh;

	u32 kern_flags;

};



DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);

struct bpf_net_context {

	struct bpf_redirect_info ri;

};



/* flags for bpf_redirect_info kern_flags */

#define BPF_RI_F_RF_NO_DIRECT	BIT(0)	/* no napi_direct on return_frame */

static inline struct bpf_net_context *bpf_net_ctx_set(struct bpf_net_context *bpf_net_ctx)

{

	struct task_struct *tsk = current;



	if (tsk->bpf_net_context != NULL)

		return NULL;

	bpf_net_ctx->ri.kern_flags = 0;



	tsk->bpf_net_context = bpf_net_ctx;

	return bpf_net_ctx;

}



static inline void bpf_net_ctx_clear(struct bpf_net_context *bpf_net_ctx)

{

	if (bpf_net_ctx)

		current->bpf_net_context = NULL;

}



static inline struct bpf_net_context *bpf_net_ctx_get(void)

{

	return current->bpf_net_context;

}



static inline struct bpf_redirect_info *bpf_net_ctx_get_ri(void)

{

	struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();



	if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_RI_INIT)) {

		memset(&bpf_net_ctx->ri, 0, offsetof(struct bpf_net_context, ri.nh));

		bpf_net_ctx->ri.kern_flags |= BPF_RI_F_RI_INIT;

	}



	return &bpf_net_ctx->ri;

}



/* Compute the linear packet data range [data, data_end) which

 * will be accessed by various program types (cls_bpf, act_bpf,
@@ -1018,25 +1056,23 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, 
				       const struct bpf_insn *patch, u32 len);

int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);



void bpf_clear_redirect_map(struct bpf_map *map);



static inline bool xdp_return_frame_no_direct(void)

{

	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();



	return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT;

}



static inline void xdp_set_return_frame_no_direct(void)

{

	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();



	ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT;

}



static inline void xdp_clear_return_frame_no_direct(void)

{

	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();



	ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT;

}
@@ -1592,7 +1628,7 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde 
						   u64 flags, const u64 flag_mask,

						   void *lookup_elem(struct bpf_map *map, u32 key))

{

	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();

	const u64 action_mask = XDP_ABORTED | XDP_DROP | XDP_PASS | XDP_TX;



	/* Lower bits of the flags are used as return code on lookup failure */

include/linux/sched.h

+3 −0
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ struct bio_list; 
struct blk_plug;

struct bpf_local_storage;

struct bpf_run_ctx;

struct bpf_net_context;

struct capture_control;

struct cfs_rq;

struct fs_struct;
@@ -1509,6 +1510,8 @@ struct task_struct { 
	/* Used for BPF run context */

	struct bpf_run_ctx		*bpf_ctx;

#endif

	/* Used by BPF for per-TASK xdp storage */

	struct bpf_net_context		*bpf_net_context;



#ifdef CONFIG_GCC_PLUGIN_STACKLEAK

	unsigned long			lowest_stack;

kernel/bpf/cpumap.c

+3 −0
Original line number Diff line number Diff line
@@ -240,12 +240,14 @@ static int cpu_map_bpf_prog_run(struct bpf_cpu_map_entry *rcpu, void **frames, 
				int xdp_n, struct xdp_cpumap_stats *stats,

				struct list_head *list)

{

	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;

	int nframes;



	if (!rcpu->prog)

		return xdp_n;



	rcu_read_lock_bh();

	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);



	nframes = cpu_map_bpf_prog_run_xdp(rcpu, frames, xdp_n, stats);
@@ -255,6 +257,7 @@ static int cpu_map_bpf_prog_run(struct bpf_cpu_map_entry *rcpu, void **frames, 
	if (unlikely(!list_empty(list)))

		cpu_map_bpf_prog_run_skb(rcpu, list, stats);



	bpf_net_ctx_clear(bpf_net_ctx);

	rcu_read_unlock_bh(); /* resched point, may call do_softirq() */



	return nframes;

kernel/bpf/devmap.c

+8 −1
Original line number Diff line number Diff line
@@ -196,7 +196,14 @@ static void dev_map_free(struct bpf_map *map) 
	list_del_rcu(&dtab->list);

	spin_unlock(&dev_map_lock);



	bpf_clear_redirect_map(map);

	/* bpf_redirect_info->map is assigned in __bpf_xdp_redirect_map()

	 * during NAPI callback and cleared after the XDP redirect. There is no

	 * explicit RCU read section which protects bpf_redirect_info->map but

	 * local_bh_disable() also marks the beginning an RCU section. This

	 * makes the complete softirq callback RCU protected. Thus after

	 * following synchronize_rcu() there no bpf_redirect_info->map == map

	 * assignment.

	 */

	synchronize_rcu();



	/* Make sure prior __dev_map_entry_free() have completed. */

kernel/fork.c

+1 −0
Original line number Diff line number Diff line
@@ -2355,6 +2355,7 @@ __latent_entropy struct task_struct *copy_process( 
	RCU_INIT_POINTER(p->bpf_storage, NULL);

	p->bpf_ctx = NULL;

#endif

	p->bpf_net_context =  NULL;



	/* Perform scheduler related setup. Assign this task to a CPU. */

	retval = sched_fork(clone_flags, p);