Commit faadc69a authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'add-kernel-symbol-for-struct_ops-trampoline' 

Xu Kuohai says:

====================
Add kernel symbol for struct_ops trampoline.

Without kernel symbol for struct_ops trampoline, the unwinder may
produce unexpected stacktraces. For example, the x86 ORC and FP
unwinder stops stacktrace on a struct_ops trampoline address since
there is no kernel symbol for the address.

v4:
- Add a separate cleanup patch to remove unused member rcu from
  bpf_struct_ops_map (patch 1)
- Use funcs_cnt instead of btf_type_vlen(vt) for links memory
  calculation in .map_mem_usage (patch 2)
- Include ksyms[] memory in map_mem_usage (patch 3)
- Various fixes in patch 3 (Thanks to Martin)

v3: https://lore.kernel.org/bpf/20241111121641.2679885-1-xukuohai@huaweicloud.com/
- Add a separate cleanup patch to replace links_cnt with funcs_cnt
- Allocate ksyms on-demand in update_elem() to stay with the links
  allocation way
- Set ksym name to prog__<struct_ops_name>_<member_name>

v2: https://lore.kernel.org/bpf/20241101111948.1570547-1-xukuohai@huaweicloud.com/
- Refine the commit message for clarity and fix a test bot warning

v1: https://lore.kernel.org/bpf/20241030111533.907289-1-xukuohai@huaweicloud.com/


====================

Reviewed-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20241112145849.3436772-1-xukuohai@huaweicloud.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents c1bc51f8 7c8ce4ff

include/linux/bpf.h

+2 −1
Original line number Diff line number Diff line
@@ -1402,7 +1402,8 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func 
void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,

				struct bpf_prog *to);

/* Called only from JIT-enabled code, so there's no need for stubs. */

void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);

void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);

void bpf_image_ksym_add(struct bpf_ksym *ksym);

void bpf_image_ksym_del(struct bpf_ksym *ksym);

void bpf_ksym_add(struct bpf_ksym *ksym);

void bpf_ksym_del(struct bpf_ksym *ksym);

kernel/bpf/bpf_struct_ops.c

+103 −12
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ struct bpf_struct_ops_value { 


struct bpf_struct_ops_map {

	struct bpf_map map;

	struct rcu_head rcu;

	const struct bpf_struct_ops_desc *st_ops_desc;

	/* protect map_update */

	struct mutex lock;
@@ -32,7 +31,9 @@ struct bpf_struct_ops_map { 
	 * (in kvalue.data).

	 */

	struct bpf_link **links;

	u32 links_cnt;

	/* ksyms for bpf trampolines */

	struct bpf_ksym **ksyms;

	u32 funcs_cnt;

	u32 image_pages_cnt;

	/* image_pages is an array of pages that has all the trampolines

	 * that stores the func args before calling the bpf_prog.
@@ -481,13 +482,13 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) 
{

	u32 i;



	for (i = 0; i < st_map->links_cnt; i++) {

		if (st_map->links[i]) {

	for (i = 0; i < st_map->funcs_cnt; i++) {

		if (!st_map->links[i])

			break;

		bpf_link_put(st_map->links[i]);

		st_map->links[i] = NULL;

	}

}

}



static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)

{
@@ -586,6 +587,49 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, 
	return 0;

}



static void bpf_struct_ops_ksym_init(const char *tname, const char *mname,

				     void *image, unsigned int size,

				     struct bpf_ksym *ksym)

{

	snprintf(ksym->name, KSYM_NAME_LEN, "bpf__%s_%s", tname, mname);

	INIT_LIST_HEAD_RCU(&ksym->lnode);

	bpf_image_ksym_init(image, size, ksym);

}



static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)

{

	u32 i;



	for (i = 0; i < st_map->funcs_cnt; i++) {

		if (!st_map->ksyms[i])

			break;

		bpf_image_ksym_add(st_map->ksyms[i]);

	}

}



static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)

{

	u32 i;



	for (i = 0; i < st_map->funcs_cnt; i++) {

		if (!st_map->ksyms[i])

			break;

		bpf_image_ksym_del(st_map->ksyms[i]);

	}

}



static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)

{

	u32 i;



	for (i = 0; i < st_map->funcs_cnt; i++) {

		if (!st_map->ksyms[i])

			break;

		kfree(st_map->ksyms[i]);

		st_map->ksyms[i] = NULL;

	}

}



static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,

					   void *value, u64 flags)

{
@@ -601,6 +645,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	int prog_fd, err;

	u32 i, trampoline_start, image_off = 0;

	void *cur_image = NULL, *image = NULL;

	struct bpf_link **plink;

	struct bpf_ksym **pksym;

	const char *tname, *mname;



	if (flags)

		return -EINVAL;
@@ -639,14 +686,19 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	udata = &uvalue->data;

	kdata = &kvalue->data;



	plink = st_map->links;

	pksym = st_map->ksyms;

	tname = btf_name_by_offset(st_map->btf, t->name_off);

	module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);

	for_each_member(i, t, member) {

		const struct btf_type *mtype, *ptype;

		struct bpf_prog *prog;

		struct bpf_tramp_link *link;

		struct bpf_ksym *ksym;

		u32 moff;



		moff = __btf_member_bit_offset(t, member) / 8;

		mname = btf_name_by_offset(st_map->btf, member->name_off);

		ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);

		if (ptype == module_type) {

			if (*(void **)(udata + moff))
@@ -714,7 +766,14 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
		}

		bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,

			      &bpf_struct_ops_link_lops, prog);

		st_map->links[i] = &link->link;

		*plink++ = &link->link;



		ksym = kzalloc(sizeof(*ksym), GFP_USER);

		if (!ksym) {

			err = -ENOMEM;

			goto reset_unlock;

		}

		*pksym++ = ksym;



		trampoline_start = image_off;

		err = bpf_struct_ops_prepare_trampoline(tlinks, link,
@@ -735,6 +794,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 


		/* put prog_id to udata */

		*(unsigned long *)(udata + moff) = prog->aux->id;



		/* init ksym for this trampoline */

		bpf_struct_ops_ksym_init(tname, mname,

					 image + trampoline_start,

					 image_off - trampoline_start,

					 ksym);

	}



	if (st_ops->validate) {
@@ -783,6 +848,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
	 */



reset_unlock:

	bpf_struct_ops_map_free_ksyms(st_map);

	bpf_struct_ops_map_free_image(st_map);

	bpf_struct_ops_map_put_progs(st_map);

	memset(uvalue, 0, map->value_size);
@@ -790,6 +856,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, 
unlock:

	kfree(tlinks);

	mutex_unlock(&st_map->lock);

	if (!err)

		bpf_struct_ops_map_add_ksyms(st_map);

	return err;

}
@@ -849,7 +917,10 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) 


	if (st_map->links)

		bpf_struct_ops_map_put_progs(st_map);

	if (st_map->ksyms)

		bpf_struct_ops_map_free_ksyms(st_map);

	bpf_map_area_free(st_map->links);

	bpf_map_area_free(st_map->ksyms);

	bpf_struct_ops_map_free_image(st_map);

	bpf_map_area_free(st_map->uvalue);

	bpf_map_area_free(st_map);
@@ -866,6 +937,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) 
	if (btf_is_module(st_map->btf))

		module_put(st_map->st_ops_desc->st_ops->owner);



	bpf_struct_ops_map_del_ksyms(st_map);



	/* The struct_ops's function may switch to another struct_ops.

	 *

	 * For example, bpf_tcp_cc_x->init() may switch to
@@ -895,6 +968,19 @@ static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) 
	return 0;

}



static u32 count_func_ptrs(const struct btf *btf, const struct btf_type *t)

{

	int i;

	u32 count;

	const struct btf_member *member;



	count = 0;

	for_each_member(i, t, member)

		if (btf_type_resolve_func_ptr(btf, member->type, NULL))

			count++;

	return count;

}



static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)

{

	const struct bpf_struct_ops_desc *st_ops_desc;
@@ -961,11 +1047,15 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) 
	map = &st_map->map;



	st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);

	st_map->links_cnt = btf_type_vlen(t);

	st_map->funcs_cnt = count_func_ptrs(btf, t);

	st_map->links =

		bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *),

		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),

				   NUMA_NO_NODE);



	st_map->ksyms =

		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),

				   NUMA_NO_NODE);

	if (!st_map->uvalue || !st_map->links) {

	if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {

		ret = -ENOMEM;

		goto errout_free;

	}
@@ -994,7 +1084,8 @@ static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) 
	usage = sizeof(*st_map) +

			vt->size - sizeof(struct bpf_struct_ops_value);

	usage += vt->size;

	usage += btf_type_vlen(vt) * sizeof(struct bpf_links *);

	usage += st_map->funcs_cnt * sizeof(struct bpf_link *);

	usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);

	usage += PAGE_SIZE;

	return usage;

}

kernel/bpf/dispatcher.c

+2 −1
Original line number Diff line number Diff line
@@ -154,7 +154,8 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 
			d->image = NULL;

			goto out;

		}

		bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym);

		bpf_image_ksym_init(d->image, PAGE_SIZE, &d->ksym);

		bpf_image_ksym_add(&d->ksym);

	}



	prev_num_progs = d->num_progs;

kernel/bpf/trampoline.c

+7 −2
Original line number Diff line number Diff line
@@ -115,10 +115,14 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) 
		(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);

}



void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym)

void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)

{

	ksym->start = (unsigned long) data;

	ksym->end = ksym->start + size;

}



void bpf_image_ksym_add(struct bpf_ksym *ksym)

{

	bpf_ksym_add(ksym);

	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,

			   PAGE_SIZE, false, ksym->name);
@@ -377,7 +381,8 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) 
	ksym = &im->ksym;

	INIT_LIST_HEAD_RCU(&ksym->lnode);

	snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);

	bpf_image_ksym_add(image, size, ksym);

	bpf_image_ksym_init(image, size, ksym);

	bpf_image_ksym_add(ksym);

	return im;



out_free_image: