Commit 93265a0b authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'bpf-resilient-split-btf-followups' 

Alan Maguire says:

====================
bpf: resilient split BTF followups

Follow-up to resilient split BTF series [1],

- cleaning up libbpf relocation code (patch 1);
- adding 'struct module' support for base BTF data (patch 2);
- splitting out field iteration code into separate file (patch 3);
- sharing libbpf relocation code with the kernel (patch 4);
- adding a kbuild --btf_features flag to generate distilled base
  BTF in the module-specific case where KBUILD_EXTMOD is true
  (patch 5); and
- adding test coverage for module-based kfunc dtor (patch 6)

Generation of distilled base BTF for modules requires the pahole patch
at [2], but without it we just won't get distilled base BTF (and thus BTF
relocation on module load) for bpf_testmod.ko.

Changes since v1 [3]:

- fixed line lengths and made comparison an explicit == 0 (Andrii, patch 1)
- moved btf_iter.c changes to separate patch (Andrii, patch 3)
- grouped common targets in kernel/bpf/Makefile (Andrii, patch 4)
- updated bpf_testmod ctx alloc to use GFP_ATOMIC, and updated dtor
  selftest to use map-based dtor cleanup (Eduard, patch 6)

[1] https://lore.kernel.org/bpf/20240613095014.357981-1-alan.maguire@oracle.com/
[2] https://lore.kernel.org/bpf/20240517102714.4072080-1-alan.maguire@oracle.com/
[3] https://lore.kernel.org/bpf/20240618162449.809994-1-alan.maguire@oracle.com/
====================

Link: https://lore.kernel.org/r/20240620091733.1967885-1-alan.maguire@oracle.com


Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents cd387ce5 47a8cf0c

include/linux/btf.h

+64 −0
Original line number Diff line number Diff line
@@ -140,6 +140,7 @@ extern const struct file_operations btf_fops; 
const char *btf_get_name(const struct btf *btf);

void btf_get(struct btf *btf);

void btf_put(struct btf *btf);

const struct btf_header *btf_header(const struct btf *btf);

int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz);

struct btf *btf_get_by_fd(int fd);

int btf_get_info_by_fd(const struct btf *btf,
@@ -212,8 +213,10 @@ int btf_get_fd_by_id(u32 id); 
u32 btf_obj_id(const struct btf *btf);

bool btf_is_kernel(const struct btf *btf);

bool btf_is_module(const struct btf *btf);

bool btf_is_vmlinux(const struct btf *btf);

struct module *btf_try_get_module(const struct btf *btf);

u32 btf_nr_types(const struct btf *btf);

struct btf *btf_base_btf(const struct btf *btf);

bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,

			   const struct btf_member *m,

			   u32 expected_offset, u32 expected_size);
@@ -339,6 +342,11 @@ static inline u8 btf_int_offset(const struct btf_type *t) 
	return BTF_INT_OFFSET(*(u32 *)(t + 1));

}



static inline __u8 btf_int_bits(const struct btf_type *t)

{

	return BTF_INT_BITS(*(__u32 *)(t + 1));

}



static inline bool btf_type_is_scalar(const struct btf_type *t)

{

	return btf_type_is_int(t) || btf_type_is_enum(t);
@@ -478,6 +486,11 @@ static inline struct btf_param *btf_params(const struct btf_type *t) 
	return (struct btf_param *)(t + 1);

}



static inline struct btf_decl_tag *btf_decl_tag(const struct btf_type *t)

{

	return (struct btf_decl_tag *)(t + 1);

}



static inline int btf_id_cmp_func(const void *a, const void *b)

{

	const int *pa = a, *pb = b;
@@ -515,9 +528,38 @@ static inline const struct bpf_struct_ops_desc *bpf_struct_ops_find(struct btf * 
}

#endif



enum btf_field_iter_kind {

	BTF_FIELD_ITER_IDS,

	BTF_FIELD_ITER_STRS,

};



struct btf_field_desc {

	/* once-per-type offsets */

	int t_off_cnt, t_offs[2];

	/* member struct size, or zero, if no members */

	int m_sz;

	/* repeated per-member offsets */

	int m_off_cnt, m_offs[1];

};



struct btf_field_iter {

	struct btf_field_desc desc;

	void *p;

	int m_idx;

	int off_idx;

	int vlen;

};



#ifdef CONFIG_BPF_SYSCALL

const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);

void btf_set_base_btf(struct btf *btf, const struct btf *base_btf);

int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids);

int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t,

			enum btf_field_iter_kind iter_kind);

__u32 *btf_field_iter_next(struct btf_field_iter *it);



const char *btf_name_by_offset(const struct btf *btf, u32 offset);

const char *btf_str_by_offset(const struct btf *btf, u32 offset);

struct btf *btf_parse_vmlinux(void);

struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog);

u32 *btf_kfunc_id_set_contains(const struct btf *btf, u32 kfunc_btf_id,
@@ -544,6 +586,28 @@ static inline const struct btf_type *btf_type_by_id(const struct btf *btf, 
{

	return NULL;

}



static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)

{

}



static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf,

			       __u32 **map_ids)

{

	return -EOPNOTSUPP;

}



static inline int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t,

				      enum btf_field_iter_kind iter_kind)

{

	return -EOPNOTSUPP;

}



static inline __u32 *btf_field_iter_next(struct btf_field_iter *it)

{

	return NULL;

}



static inline const char *btf_name_by_offset(const struct btf *btf,

					     u32 offset)

{

include/linux/module.h

+2 −0
Original line number Diff line number Diff line
@@ -509,7 +509,9 @@ struct module { 
#endif

#ifdef CONFIG_DEBUG_INFO_BTF_MODULES

	unsigned int btf_data_size;

	unsigned int btf_base_data_size;

	void *btf_data;

	void *btf_base_data;

#endif

#ifdef CONFIG_JUMP_LABEL

	struct jump_entry *jump_entries;

kernel/bpf/Makefile

+7 −1
Original line number Diff line number Diff line
@@ -50,5 +50,11 @@ endif 
obj-$(CONFIG_BPF_PRELOAD) += preload/



obj-$(CONFIG_BPF_SYSCALL) += relo_core.o

$(obj)/relo_core.o: $(srctree)/tools/lib/bpf/relo_core.c FORCE

obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o

obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o



# Some source files are common to libbpf.

vpath %.c $(srctree)/kernel/bpf:$(srctree)/tools/lib/bpf



$(obj)/%.o: %.c FORCE

	$(call if_changed_rule,cc_o_c)

kernel/bpf/btf.c

+124 −52
Original line number Diff line number Diff line
@@ -274,6 +274,7 @@ struct btf { 
	u32 start_str_off; /* first string offset (0 for base BTF) */

	char name[MODULE_NAME_LEN];

	bool kernel_btf;

	__u32 *base_id_map; /* map from distilled base BTF -> vmlinux BTF ids */

};



enum verifier_phase {
@@ -530,6 +531,11 @@ static bool btf_type_is_decl_tag_target(const struct btf_type *t) 
	       btf_type_is_var(t) || btf_type_is_typedef(t);

}



bool btf_is_vmlinux(const struct btf *btf)

{

	return btf->kernel_btf && !btf->base_btf;

}



u32 btf_nr_types(const struct btf *btf)

{

	u32 total = 0;
@@ -772,7 +778,7 @@ static bool __btf_name_char_ok(char c, bool first) 
	return true;

}



static const char *btf_str_by_offset(const struct btf *btf, u32 offset)

const char *btf_str_by_offset(const struct btf *btf, u32 offset)

{

	while (offset < btf->start_str_off)

		btf = btf->base_btf;
@@ -1670,14 +1676,8 @@ static void btf_free_kfunc_set_tab(struct btf *btf) 


	if (!tab)

		return;

	/* For module BTF, we directly assign the sets being registered, so

	 * there is nothing to free except kfunc_set_tab.

	 */

	if (btf_is_module(btf))

		goto free_tab;

	for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++)

		kfree(tab->sets[hook]);

free_tab:

	kfree(tab);

	btf->kfunc_set_tab = NULL;

}
@@ -1735,7 +1735,12 @@ static void btf_free(struct btf *btf) 
	kvfree(btf->types);

	kvfree(btf->resolved_sizes);

	kvfree(btf->resolved_ids);

	/* vmlinux does not allocate btf->data, it simply points it at

	 * __start_BTF.

	 */

	if (!btf_is_vmlinux(btf))

		kvfree(btf->data);

	kvfree(btf->base_id_map);

	kfree(btf);

}
@@ -1764,6 +1769,23 @@ void btf_put(struct btf *btf) 
	}

}



struct btf *btf_base_btf(const struct btf *btf)

{

	return btf->base_btf;

}



const struct btf_header *btf_header(const struct btf *btf)

{

	return &btf->hdr;

}



void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)

{

	btf->base_btf = (struct btf *)base_btf;

	btf->start_id = btf_nr_types(base_btf);

	btf->start_str_off = base_btf->hdr.str_len;

}



static int env_resolve_init(struct btf_verifier_env *env)

{

	struct btf *btf = env->btf;
@@ -6083,23 +6105,15 @@ int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_ty 
BTF_ID_LIST(bpf_ctx_convert_btf_id)

BTF_ID(struct, bpf_ctx_convert)



struct btf *btf_parse_vmlinux(void)

static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name,

				  void *data, unsigned int data_size)

{

	struct btf_verifier_env *env = NULL;

	struct bpf_verifier_log *log;

	struct btf *btf = NULL;

	int err;



	if (!IS_ENABLED(CONFIG_DEBUG_INFO_BTF))

		return ERR_PTR(-ENOENT);



	env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);

	if (!env)

		return ERR_PTR(-ENOMEM);



	log = &env->log;

	log->level = BPF_LOG_KERNEL;



	btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);

	if (!btf) {

		err = -ENOMEM;
@@ -6107,10 +6121,10 @@ struct btf *btf_parse_vmlinux(void) 
	}

	env->btf = btf;



	btf->data = __start_BTF;

	btf->data_size = __stop_BTF - __start_BTF;

	btf->data = data;

	btf->data_size = data_size;

	btf->kernel_btf = true;

	snprintf(btf->name, sizeof(btf->name), "vmlinux");

	snprintf(btf->name, sizeof(btf->name), "%s", name);



	err = btf_parse_hdr(env);

	if (err)
@@ -6130,20 +6144,11 @@ struct btf *btf_parse_vmlinux(void) 
	if (err)

		goto errout;



	/* btf_parse_vmlinux() runs under bpf_verifier_lock */

	bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);



	refcount_set(&btf->refcnt, 1);



	err = btf_alloc_id(btf);

	if (err)

		goto errout;



	btf_verifier_env_free(env);

	return btf;



errout:

	btf_verifier_env_free(env);

	if (btf) {

		kvfree(btf->types);

		kfree(btf);
@@ -6151,19 +6156,61 @@ struct btf *btf_parse_vmlinux(void) 
	return ERR_PTR(err);

}



struct btf *btf_parse_vmlinux(void)

{

	struct btf_verifier_env *env = NULL;

	struct bpf_verifier_log *log;

	struct btf *btf;

	int err;



	env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);

	if (!env)

		return ERR_PTR(-ENOMEM);



	log = &env->log;

	log->level = BPF_LOG_KERNEL;

	btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF);

	if (IS_ERR(btf))

		goto err_out;



	/* btf_parse_vmlinux() runs under bpf_verifier_lock */

	bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);

	err = btf_alloc_id(btf);

	if (err) {

		btf_free(btf);

		btf = ERR_PTR(err);

	}

err_out:

	btf_verifier_env_free(env);

	return btf;

}



#ifdef CONFIG_DEBUG_INFO_BTF_MODULES



static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size)

/* If .BTF_ids section was created with distilled base BTF, both base and

 * split BTF ids will need to be mapped to actual base/split ids for

 * BTF now that it has been relocated.

 */

static __u32 btf_relocate_id(const struct btf *btf, __u32 id)

{

	if (!btf->base_btf || !btf->base_id_map)

		return id;

	return btf->base_id_map[id];

}



static struct btf *btf_parse_module(const char *module_name, const void *data,

				    unsigned int data_size, void *base_data,

				    unsigned int base_data_size)

{

	struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL;

	struct btf_verifier_env *env = NULL;

	struct bpf_verifier_log *log;

	struct btf *btf = NULL, *base_btf;

	int err;

	int err = 0;



	base_btf = bpf_get_btf_vmlinux();

	if (IS_ERR(base_btf))

		return base_btf;

	if (!base_btf)

	vmlinux_btf = bpf_get_btf_vmlinux();

	if (IS_ERR(vmlinux_btf))

		return vmlinux_btf;

	if (!vmlinux_btf)

		return ERR_PTR(-EINVAL);



	env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
@@ -6173,6 +6220,16 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, u 
	log = &env->log;

	log->level = BPF_LOG_KERNEL;



	if (base_data) {

		base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size);

		if (IS_ERR(base_btf)) {

			err = PTR_ERR(base_btf);

			goto errout;

		}

	} else {

		base_btf = vmlinux_btf;

	}



	btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);

	if (!btf) {

		err = -ENOMEM;
@@ -6212,12 +6269,22 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, u 
	if (err)

		goto errout;



	if (base_btf != vmlinux_btf) {

		err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map);

		if (err)

			goto errout;

		btf_free(base_btf);

		base_btf = vmlinux_btf;

	}



	btf_verifier_env_free(env);

	refcount_set(&btf->refcnt, 1);

	return btf;



errout:

	btf_verifier_env_free(env);

	if (base_btf != vmlinux_btf)

		btf_free(base_btf);

	if (btf) {

		kvfree(btf->data);

		kvfree(btf->types);
@@ -7770,7 +7837,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, 
			err = -ENOMEM;

			goto out;

		}

		btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size);

		btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size,

				       mod->btf_base_data, mod->btf_base_data_size);

		if (IS_ERR(btf)) {

			kfree(btf_mod);

			if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) {
@@ -8094,7 +8162,7 @@ static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, 
	bool add_filter = !!kset->filter;

	struct btf_kfunc_set_tab *tab;

	struct btf_id_set8 *set;

	u32 set_cnt;

	u32 set_cnt, i;

	int ret;



	if (hook >= BTF_KFUNC_HOOK_MAX) {
@@ -8140,21 +8208,15 @@ static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, 
		goto end;

	}



	/* We don't need to allocate, concatenate, and sort module sets, because

	 * only one is allowed per hook. Hence, we can directly assign the

	 * pointer and return.

	 */

	if (!vmlinux_set) {

		tab->sets[hook] = add_set;

		goto do_add_filter;

	}



	/* In case of vmlinux sets, there may be more than one set being

	 * registered per hook. To create a unified set, we allocate a new set

	 * and concatenate all individual sets being registered. While each set

	 * is individually sorted, they may become unsorted when concatenated,

	 * hence re-sorting the final set again is required to make binary

	 * searching the set using btf_id_set8_contains function work.

	 *

	 * For module sets, we need to allocate as we may need to relocate

	 * BTF ids.

	 */

	set_cnt = set ? set->cnt : 0;
@@ -8184,11 +8246,14 @@ static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, 


	/* Concatenate the two sets */

	memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0]));

	/* Now that the set is copied, update with relocated BTF ids */

	for (i = set->cnt; i < set->cnt + add_set->cnt; i++)

		set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id);



	set->cnt += add_set->cnt;



	sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL);



do_add_filter:

	if (add_filter) {

		hook_filter = &tab->hook_filters[hook];

		hook_filter->filters[hook_filter->nr_filters++] = kset->filter;
@@ -8308,7 +8373,7 @@ static int __register_btf_kfunc_id_set(enum btf_kfunc_hook hook, 
		return PTR_ERR(btf);



	for (i = 0; i < kset->set->cnt; i++) {

		ret = btf_check_kfunc_protos(btf, kset->set->pairs[i].id,

		ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id),

					     kset->set->pairs[i].flags);

		if (ret)

			goto err_out;
@@ -8372,7 +8437,7 @@ static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc 
	u32 nr_args, i;



	for (i = 0; i < cnt; i++) {

		dtor_btf_id = dtors[i].kfunc_btf_id;

		dtor_btf_id = btf_relocate_id(btf, dtors[i].kfunc_btf_id);



		dtor_func = btf_type_by_id(btf, dtor_btf_id);

		if (!dtor_func || !btf_type_is_func(dtor_func))
@@ -8407,7 +8472,7 @@ int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_c 
{

	struct btf_id_dtor_kfunc_tab *tab;

	struct btf *btf;

	u32 tab_cnt;

	u32 tab_cnt, i;

	int ret;



	btf = btf_get_module_btf(owner);
@@ -8458,6 +8523,13 @@ int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_c 
	btf->dtor_kfunc_tab = tab;



	memcpy(tab->dtors + tab->cnt, dtors, add_cnt * sizeof(tab->dtors[0]));



	/* remap BTF ids based on BTF relocation (if any) */

	for (i = tab_cnt; i < tab_cnt + add_cnt; i++) {

		tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id);

		tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id);

	}



	tab->cnt += add_cnt;



	sort(tab->dtors, tab->cnt, sizeof(tab->dtors[0]), btf_id_cmp_func, NULL);

kernel/module/main.c

+4 −1
Original line number Diff line number Diff line
@@ -2166,6 +2166,8 @@ static int find_module_sections(struct module *mod, struct load_info *info) 
#endif

#ifdef CONFIG_DEBUG_INFO_BTF_MODULES

	mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);

	mod->btf_base_data = any_section_objs(info, ".BTF.base", 1,

					      &mod->btf_base_data_size);

#endif

#ifdef CONFIG_JUMP_LABEL

	mod->jump_entries = section_objs(info, "__jump_table",
@@ -2590,8 +2592,9 @@ static noinline int do_init_module(struct module *mod) 
	}



#ifdef CONFIG_DEBUG_INFO_BTF_MODULES

	/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */

	/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointers */

	mod->btf_data = NULL;

	mod->btf_base_data = NULL;

#endif

	/*

	 * We want to free module_init, but be aware that kallsyms may be