Commit 8ca77b8f authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-fix-tailcall-infinite-loop-caused-by-freplace' 

Leon Hwang says:

====================
bpf: Fix tailcall infinite loop caused by freplace

Previously, I addressed a tailcall infinite loop issue related to
trampolines[0].

In this patchset, I resolve a similar issue where a tailcall infinite loop
can occur due to the combination of tailcalls and freplace programs. The
fix prevents adding extended programs to the prog_array map and blocks the
extension of a tail callee program with freplace.

Key changes:

1. If a program or its subprogram has been extended by an freplace program,
   it can no longer be updated to a prog_array map.
2. If a program has been added to a prog_array map, neither it nor its
   subprograms can be extended by an freplace program.

Additionally, an extension program should not be tailcalled. As a result,
return -EINVAL if the program has a type of BPF_PROG_TYPE_EXT when adding
it to a prog_array map.

Changes:
v7 -> v8:
  * Address comment from Alexei:
    * guard(mutex) should not hold range all the way through
      bpf_arch_text_poke().
  * Address suggestion from Xu Kuohai:
    * Extension prog should not be tailcalled independently.

v6 -> v7:
  * Address comments from Alexei:
    * Rewrite commit message more imperative and consice with AI.
    * Extend bpf_trampoline_link_prog() and bpf_trampoline_unlink_prog()
      to link and unlink target prog for freplace prog.
    * Use guard(mutex)(&tgt_prog->aux->ext_mutex) instead of
      mutex_lock()&mutex_unlock() pair.
  * Address comment from Eduard:
    * Remove misplaced "Reported-by" and "Closes" tags.

v5 -> v6:
  * Fix a build warning reported by kernel test robot.

v4 -> v5:
  * Move code of linking/unlinking target prog of freplace to trampoline.c.
  * Address comments from Alexei:
    * Change type of prog_array_member_cnt to u64.
    * Combine two patches to one.

v3 -> v4:
  * Address comments from Eduard:
    * Rename 'tail_callee_cnt' to 'prog_array_member_cnt'.
    * Add comment to 'prog_array_member_cnt'.
    * Use a mutex to protect 'is_extended' and 'prog_array_member_cnt'.

v2 -> v3:
  * Address comments from Alexei:
    * Stop hacking JIT.
    * Prevent the specific use case at attach/update time.

v1 -> v2:
  * Address comment from Eduard:
    * Explain why nop5 and xor/nop3 are swapped at prologue.
  * Address comment from Alexei:
    * Disallow attaching tail_call_reachable freplace prog to
      not-tail_call_reachable target in verifier.
  * Update "bpf, arm64: Fix tailcall infinite loop caused by freplace" with
    latest arm64 JIT code.

Links:
[0] https://lore.kernel.org/bpf/20230912150442.2009-1-hffilwlqm@gmail.com/
====================

Link: https://lore.kernel.org/r/20241015150207.70264-1-leon.hwang@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents f987a640 021611d3

include/linux/bpf.h

+13 −4
Original line number Diff line number Diff line
@@ -1292,8 +1292,12 @@ void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len); 
bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);



#ifdef CONFIG_BPF_JIT

int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);

int bpf_trampoline_link_prog(struct bpf_tramp_link *link,

			     struct bpf_trampoline *tr,

			     struct bpf_prog *tgt_prog);

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,

			       struct bpf_trampoline *tr,

			       struct bpf_prog *tgt_prog);

struct bpf_trampoline *bpf_trampoline_get(u64 key,

					  struct bpf_attach_target_info *tgt_info);

void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -1374,12 +1378,14 @@ void bpf_jit_uncharge_modmem(u32 size); 
bool bpf_prog_has_trampoline(const struct bpf_prog *prog);

#else

static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,

					   struct bpf_trampoline *tr)

					   struct bpf_trampoline *tr,

					   struct bpf_prog *tgt_prog)

{

	return -ENOTSUPP;

}

static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,

					     struct bpf_trampoline *tr)

					     struct bpf_trampoline *tr,

					     struct bpf_prog *tgt_prog)

{

	return -ENOTSUPP;

}
@@ -1483,6 +1489,9 @@ struct bpf_prog_aux { 
	bool xdp_has_frags;

	bool exception_cb;

	bool exception_boundary;

	bool is_extended; /* true if extended by freplace program */

	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */

	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */

	struct bpf_arena *arena;

	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */

	const struct btf_type *attach_func_proto;

kernel/bpf/arraymap.c

+24 −2
Original line number Diff line number Diff line
@@ -947,22 +947,44 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, 
				   struct file *map_file, int fd)

{

	struct bpf_prog *prog = bpf_prog_get(fd);

	bool is_extended;



	if (IS_ERR(prog))

		return prog;



	if (!bpf_prog_map_compatible(map, prog)) {

	if (prog->type == BPF_PROG_TYPE_EXT ||

	    !bpf_prog_map_compatible(map, prog)) {

		bpf_prog_put(prog);

		return ERR_PTR(-EINVAL);

	}



	mutex_lock(&prog->aux->ext_mutex);

	is_extended = prog->aux->is_extended;

	if (!is_extended)

		prog->aux->prog_array_member_cnt++;

	mutex_unlock(&prog->aux->ext_mutex);

	if (is_extended) {

		/* Extended prog can not be tail callee. It's to prevent a

		 * potential infinite loop like:

		 * tail callee prog entry -> tail callee prog subprog ->

		 * freplace prog entry --tailcall-> tail callee prog entry.

		 */

		bpf_prog_put(prog);

		return ERR_PTR(-EBUSY);

	}



	return prog;

}



static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)

{

	struct bpf_prog *prog = ptr;



	mutex_lock(&prog->aux->ext_mutex);

	prog->aux->prog_array_member_cnt--;

	mutex_unlock(&prog->aux->ext_mutex);

	/* bpf_prog is freed after one RCU or tasks trace grace period */

	bpf_prog_put(ptr);

	bpf_prog_put(prog);

}



static u32 prog_fd_array_sys_lookup_elem(void *ptr)

kernel/bpf/core.c

+1 −0
Original line number Diff line number Diff line
@@ -131,6 +131,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag 
	INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);

#endif

	mutex_init(&fp->aux->used_maps_mutex);

	mutex_init(&fp->aux->ext_mutex);

	mutex_init(&fp->aux->dst_mutex);



	return fp;

kernel/bpf/syscall.c

+4 −3
Original line number Diff line number Diff line
@@ -3214,7 +3214,8 @@ static void bpf_tracing_link_release(struct bpf_link *link) 
		container_of(link, struct bpf_tracing_link, link.link);



	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,

						tr_link->trampoline));

						tr_link->trampoline,

						tr_link->tgt_prog));



	bpf_trampoline_put(tr_link->trampoline);
@@ -3429,7 +3430,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, 
	if (err)

		goto out_unlock;



	err = bpf_trampoline_link_prog(&link->link, tr);

	err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog);

	if (err) {

		bpf_link_cleanup(&link_primer);

		link = NULL;

kernel/bpf/trampoline.c

+39 −8
Original line number Diff line number Diff line
@@ -523,7 +523,27 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) 
	}

}



static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)

{

	struct bpf_prog_aux *aux = tgt_prog->aux;



	guard(mutex)(&aux->ext_mutex);

	if (aux->prog_array_member_cnt)

		/* Program extensions can not extend target prog when the target

		 * prog has been updated to any prog_array map as tail callee.

		 * It's to prevent a potential infinite loop like:

		 * tgt prog entry -> tgt prog subprog -> freplace prog entry

		 * --tailcall-> tgt prog entry.

		 */

		return -EBUSY;



	aux->is_extended = true;

	return 0;

}



static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,

				      struct bpf_trampoline *tr,

				      struct bpf_prog *tgt_prog)

{

	enum bpf_tramp_prog_type kind;

	struct bpf_tramp_link *link_exiting;
@@ -544,6 +564,9 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr 
		/* Cannot attach extension if fentry/fexit are in use. */

		if (cnt)

			return -EBUSY;

		err = bpf_freplace_check_tgt_prog(tgt_prog);

		if (err)

			return err;

		tr->extension_prog = link->link.prog;

		return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,

					  link->link.prog->bpf_func);
@@ -570,17 +593,21 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr 
	return err;

}



int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

int bpf_trampoline_link_prog(struct bpf_tramp_link *link,

			     struct bpf_trampoline *tr,

			     struct bpf_prog *tgt_prog)

{

	int err;



	mutex_lock(&tr->mutex);

	err = __bpf_trampoline_link_prog(link, tr);

	err = __bpf_trampoline_link_prog(link, tr, tgt_prog);

	mutex_unlock(&tr->mutex);

	return err;

}



static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,

					struct bpf_trampoline *tr,

					struct bpf_prog *tgt_prog)

{

	enum bpf_tramp_prog_type kind;

	int err;
@@ -591,6 +618,8 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ 
		err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,

					 tr->extension_prog->bpf_func, NULL);

		tr->extension_prog = NULL;

		guard(mutex)(&tgt_prog->aux->ext_mutex);

		tgt_prog->aux->is_extended = false;

		return err;

	}

	hlist_del_init(&link->tramp_hlist);
@@ -599,12 +628,14 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ 
}



/* bpf_trampoline_unlink_prog() should never fail. */

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)

int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,

			       struct bpf_trampoline *tr,

			       struct bpf_prog *tgt_prog)

{

	int err;



	mutex_lock(&tr->mutex);

	err = __bpf_trampoline_unlink_prog(link, tr);

	err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);

	mutex_unlock(&tr->mutex);

	return err;

}
@@ -619,7 +650,7 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) 
	if (!shim_link->trampoline)

		return;



	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));

	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline, NULL));

	bpf_trampoline_put(shim_link->trampoline);

}
@@ -733,7 +764,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 
		goto err;

	}



	err = __bpf_trampoline_link_prog(&shim_link->link, tr);

	err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);

	if (err)

		goto err;