Commit 7d045249 authored by Jiri Olsa's avatar Jiri Olsa Committed by Andrii Nakryiko
Browse files

bpf: Add trampoline ip hash table 



Following changes need to lookup trampoline based on its ip address,
adding hash table for that.

Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-8-jolsa@kernel.org
parent e93672f7

include/linux/bpf.h

+5 −2
Original line number Diff line number Diff line
@@ -1329,14 +1329,17 @@ struct bpf_tramp_image { 
};



struct bpf_trampoline {

	/* hlist for trampoline_table */

	struct hlist_node hlist;

	/* hlist for trampoline_key_table */

	struct hlist_node hlist_key;

	/* hlist for trampoline_ip_table */

	struct hlist_node hlist_ip;

	struct ftrace_ops *fops;

	/* serializes access to fields of this trampoline */

	struct mutex mutex;

	refcount_t refcnt;

	u32 flags;

	u64 key;

	unsigned long ip;

	struct {

		struct btf_func_model model;

		void *addr;

kernel/bpf/trampoline.c

+19 −11
Original line number Diff line number Diff line
@@ -24,9 +24,10 @@ const struct bpf_prog_ops bpf_extension_prog_ops = { 
#define TRAMPOLINE_HASH_BITS 10

#define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)



static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE];

static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE];

static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE];



/* serializes access to trampoline_table */

/* serializes access to trampoline tables */

static DEFINE_MUTEX(trampoline_mutex);



#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
@@ -142,15 +143,15 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym) 
			   PAGE_SIZE, true, ksym->name);

}



static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)

static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)

{

	struct bpf_trampoline *tr;

	struct hlist_head *head;

	int i;



	mutex_lock(&trampoline_mutex);

	head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];

	hlist_for_each_entry(tr, head, hlist) {

	head = &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)];

	hlist_for_each_entry(tr, head, hlist_key) {

		if (tr->key == key) {

			refcount_inc(&tr->refcnt);

			goto out;
@@ -171,8 +172,12 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 
#endif



	tr->key = key;

	INIT_HLIST_NODE(&tr->hlist);

	hlist_add_head(&tr->hlist, head);

	tr->ip = ftrace_location(ip);

	INIT_HLIST_NODE(&tr->hlist_key);

	INIT_HLIST_NODE(&tr->hlist_ip);

	hlist_add_head(&tr->hlist_key, head);

	head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)];

	hlist_add_head(&tr->hlist_ip, head);

	refcount_set(&tr->refcnt, 1);

	mutex_init(&tr->mutex);

	for (i = 0; i < BPF_TRAMP_MAX; i++)
@@ -883,7 +888,7 @@ void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) 
					 prog->aux->attach_btf_id);



	bpf_lsm_find_cgroup_shim(prog, &bpf_func);

	tr = bpf_trampoline_lookup(key);

	tr = bpf_trampoline_lookup(key, 0);

	if (WARN_ON_ONCE(!tr))

		return;
@@ -903,7 +908,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, 
{

	struct bpf_trampoline *tr;



	tr = bpf_trampoline_lookup(key);

	tr = bpf_trampoline_lookup(key, tgt_info->tgt_addr);

	if (!tr)

		return NULL;
@@ -939,7 +944,8 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) 
	 * fexit progs. The fentry-only trampoline will be freed via

	 * multiple rcu callbacks.

	 */

	hlist_del(&tr->hlist);

	hlist_del(&tr->hlist_key);

	hlist_del(&tr->hlist_ip);

	if (tr->fops) {

		ftrace_free_filter(tr->fops);

		kfree(tr->fops);
@@ -1212,7 +1218,9 @@ static int __init init_trampolines(void) 
	int i;



	for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)

		INIT_HLIST_HEAD(&trampoline_table[i]);

		INIT_HLIST_HEAD(&trampoline_key_table[i]);

	for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)

		INIT_HLIST_HEAD(&trampoline_ip_table[i]);

	return 0;

}

late_initcall(init_trampolines);