Commit c85e5594 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf 

Daniel Borkmann says:

====================
pull-request: bpf 2023-12-06

We've added 4 non-merge commits during the last 6 day(s) which contain
a total of 7 files changed, 185 insertions(+), 55 deletions(-).

The main changes are:

1) Fix race found by syzkaller on prog_array_map_poke_run when
   a BPF program's kallsym symbols were still missing, from Jiri Olsa.

2) Fix BPF verifier's branch offset comparison for BPF_JMP32 | BPF_JA,
   from Yonghong Song.

3) Fix xsk's poll handling to only set mask on bound xsk sockets,
   from Yewon Choi.

* tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf:
  selftests/bpf: Add test for early update in prog_array_map_poke_run
  bpf: Fix prog_array_map_poke_run map poke update
  xsk: Skip polling event check for unbound socket
  bpf: Fix a verifier bug due to incorrect branch offset comparison with cpu=v4
====================

Link: https://lore.kernel.org/r/20231206220528.12093-1-daniel@iogearbox.net


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 0ad722bd ffed24ef

arch/x86/net/bpf_jit_comp.c

+46 −0
Original line number Diff line number Diff line
@@ -3025,3 +3025,49 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp 
#endif

	WARN(1, "verification of programs using bpf_throw should have failed\n");

}



void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,

			       struct bpf_prog *new, struct bpf_prog *old)

{

	u8 *old_addr, *new_addr, *old_bypass_addr;

	int ret;



	old_bypass_addr = old ? NULL : poke->bypass_addr;

	old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;

	new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;



	/*

	 * On program loading or teardown, the program's kallsym entry

	 * might not be in place, so we use __bpf_arch_text_poke to skip

	 * the kallsyms check.

	 */

	if (new) {

		ret = __bpf_arch_text_poke(poke->tailcall_target,

					   BPF_MOD_JUMP,

					   old_addr, new_addr);

		BUG_ON(ret < 0);

		if (!old) {

			ret = __bpf_arch_text_poke(poke->tailcall_bypass,

						   BPF_MOD_JUMP,

						   poke->bypass_addr,

						   NULL);

			BUG_ON(ret < 0);

		}

	} else {

		ret = __bpf_arch_text_poke(poke->tailcall_bypass,

					   BPF_MOD_JUMP,

					   old_bypass_addr,

					   poke->bypass_addr);

		BUG_ON(ret < 0);

		/* let other CPUs finish the execution of program

		 * so that it will not possible to expose them

		 * to invalid nop, stack unwind, nop state

		 */

		if (!ret)

			synchronize_rcu();

		ret = __bpf_arch_text_poke(poke->tailcall_target,

					   BPF_MOD_JUMP,

					   old_addr, NULL);

		BUG_ON(ret < 0);

	}

}

include/linux/bpf.h

+3 −0
Original line number Diff line number Diff line
@@ -3175,6 +3175,9 @@ enum bpf_text_poke_type { 
int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,

		       void *addr1, void *addr2);



void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,

			       struct bpf_prog *new, struct bpf_prog *old);



void *bpf_arch_text_copy(void *dst, void *src, size_t len);

int bpf_arch_text_invalidate(void *dst, size_t len);

kernel/bpf/arraymap.c

+10 −48
Original line number Diff line number Diff line
@@ -1012,11 +1012,16 @@ static void prog_array_map_poke_untrack(struct bpf_map *map, 
	mutex_unlock(&aux->poke_mutex);

}



void __weak bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,

				      struct bpf_prog *new, struct bpf_prog *old)

{

	WARN_ON_ONCE(1);

}



static void prog_array_map_poke_run(struct bpf_map *map, u32 key,

				    struct bpf_prog *old,

				    struct bpf_prog *new)

{

	u8 *old_addr, *new_addr, *old_bypass_addr;

	struct prog_poke_elem *elem;

	struct bpf_array_aux *aux;
@@ -1025,7 +1030,7 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key, 


	list_for_each_entry(elem, &aux->poke_progs, list) {

		struct bpf_jit_poke_descriptor *poke;

		int i, ret;

		int i;



		for (i = 0; i < elem->aux->size_poke_tab; i++) {

			poke = &elem->aux->poke_tab[i];
@@ -1044,21 +1049,10 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key, 
			 *    activated, so tail call updates can arrive from here

			 *    while JIT is still finishing its final fixup for

			 *    non-activated poke entries.

			 * 3) On program teardown, the program's kallsym entry gets

			 *    removed out of RCU callback, but we can only untrack

			 *    from sleepable context, therefore bpf_arch_text_poke()

			 *    might not see that this is in BPF text section and

			 *    bails out with -EINVAL. As these are unreachable since

			 *    RCU grace period already passed, we simply skip them.

			 * 4) Also programs reaching refcount of zero while patching

			 * 3) Also programs reaching refcount of zero while patching

			 *    is in progress is okay since we're protected under

			 *    poke_mutex and untrack the programs before the JIT

			 *    buffer is freed. When we're still in the middle of

			 *    patching and suddenly kallsyms entry of the program

			 *    gets evicted, we just skip the rest which is fine due

			 *    to point 3).

			 * 5) Any other error happening below from bpf_arch_text_poke()

			 *    is a unexpected bug.

			 *    buffer is freed.

			 */

			if (!READ_ONCE(poke->tailcall_target_stable))

				continue;
@@ -1068,39 +1062,7 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key, 
			    poke->tail_call.key != key)

				continue;



			old_bypass_addr = old ? NULL : poke->bypass_addr;

			old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;

			new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;



			if (new) {

				ret = bpf_arch_text_poke(poke->tailcall_target,

							 BPF_MOD_JUMP,

							 old_addr, new_addr);

				BUG_ON(ret < 0 && ret != -EINVAL);

				if (!old) {

					ret = bpf_arch_text_poke(poke->tailcall_bypass,

								 BPF_MOD_JUMP,

								 poke->bypass_addr,

								 NULL);

					BUG_ON(ret < 0 && ret != -EINVAL);

				}

			} else {

				ret = bpf_arch_text_poke(poke->tailcall_bypass,

							 BPF_MOD_JUMP,

							 old_bypass_addr,

							 poke->bypass_addr);

				BUG_ON(ret < 0 && ret != -EINVAL);

				/* let other CPUs finish the execution of program

				 * so that it will not possible to expose them

				 * to invalid nop, stack unwind, nop state

				 */

				if (!ret)

					synchronize_rcu();

				ret = bpf_arch_text_poke(poke->tailcall_target,

							 BPF_MOD_JUMP,

							 old_addr, NULL);

				BUG_ON(ret < 0 && ret != -EINVAL);

			}

			bpf_arch_poke_desc_update(poke, new, old);

		}

	}

}

kernel/bpf/core.c

+8 −4
Original line number Diff line number Diff line
@@ -371,14 +371,18 @@ static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, 
static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old,

				s32 end_new, s32 curr, const bool probe_pass)

{

	const s32 off_min = S16_MIN, off_max = S16_MAX;

	s64 off_min, off_max, off;

	s32 delta = end_new - end_old;

	s32 off;



	if (insn->code == (BPF_JMP32 | BPF_JA))

	if (insn->code == (BPF_JMP32 | BPF_JA)) {

		off = insn->imm;

	else

		off_min = S32_MIN;

		off_max = S32_MAX;

	} else {

		off = insn->off;

		off_min = S16_MIN;

		off_max = S16_MAX;

	}



	if (curr < pos && curr + off + 1 >= end_old)

		off += delta;

net/xdp/xsk.c

+2 −3
Original line number Diff line number Diff line
@@ -947,7 +947,7 @@ static __poll_t xsk_poll(struct file *file, struct socket *sock, 


	rcu_read_lock();

	if (xsk_check_common(xs))

		goto skip_tx;

		goto out;



	pool = xs->pool;
@@ -959,12 +959,11 @@ static __poll_t xsk_poll(struct file *file, struct socket *sock, 
			xsk_generic_xmit(sk);

	}



skip_tx:

	if (xs->rx && !xskq_prod_is_empty(xs->rx))

		mask |= EPOLLIN | EPOLLRDNORM;

	if (xs->tx && xsk_tx_writeable(xs))

		mask |= EPOLLOUT | EPOLLWRNORM;



out:

	rcu_read_unlock();

	return mask;

}