Commit 424f6a36 authored by Jiri Olsa's avatar Jiri Olsa Committed by Andrii Nakryiko
Browse files

bpf,x86: Use single ftrace_ops for direct calls 



Using single ftrace_ops for direct calls update instead of allocating
ftrace_ops object for each trampoline.

With single ftrace_ops object we can use update_ftrace_direct_* api
that allows multiple ip sites updates on single ftrace_ops object.

Adding HAVE_SINGLE_FTRACE_DIRECT_OPS config option to be enabled on
each arch that supports this.

At the moment we can enable this only on x86 arch, because arm relies
on ftrace_ops object representing just single trampoline image (stored
in ftrace_ops::direct_call). Archs that do not support this will continue
to use *_ftrace_direct api.

Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Reviewed-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/bpf/20251230145010.103439-10-jolsa@kernel.org
parent 956747ef

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -336,6 +336,7 @@ config X86 
	select SCHED_SMT			if SMP

	select ARCH_SUPPORTS_SCHED_CLUSTER	if SMP

	select ARCH_SUPPORTS_SCHED_MC		if SMP

	select HAVE_SINGLE_FTRACE_DIRECT_OPS	if X86_64 && DYNAMIC_FTRACE_WITH_DIRECT_CALLS



config INSTRUCTION_DECODER

	def_bool y

kernel/bpf/trampoline.c

+190 −30
Original line number Diff line number Diff line
@@ -33,12 +33,40 @@ static DEFINE_MUTEX(trampoline_mutex); 
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);



#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS

static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)

{

	struct hlist_head *head_ip;

	struct bpf_trampoline *tr;



	mutex_lock(&trampoline_mutex);

	head_ip = &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)];

	hlist_for_each_entry(tr, head_ip, hlist_ip) {

		if (tr->ip == ip)

			goto out;

	}

	tr = NULL;

out:

	mutex_unlock(&trampoline_mutex);

	return tr;

}

#else

static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip)

{

	return ops->private;

}

#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */



static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip,

				     enum ftrace_ops_cmd cmd)

{

	struct bpf_trampoline *tr = ops->private;

	struct bpf_trampoline *tr;

	int ret = 0;



	tr = direct_ops_ip_lookup(ops, ip);

	if (!tr)

		return -EINVAL;



	if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {

		/* This is called inside register_ftrace_direct_multi(), so

		 * tr->mutex is already locked.
@@ -144,6 +172,162 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym) 
			   PAGE_SIZE, true, ksym->name);

}



#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS

/*

 * We have only single direct_ops which contains all the direct call

 * sites and is the only global ftrace_ops for all trampolines.

 *

 * We use 'update_ftrace_direct_*' api for attachment.

 */

struct ftrace_ops direct_ops = {

	.ops_func = bpf_tramp_ftrace_ops_func,

};



static int direct_ops_alloc(struct bpf_trampoline *tr)

{

	tr->fops = &direct_ops;

	return 0;

}



static void direct_ops_free(struct bpf_trampoline *tr) { }



static struct ftrace_hash *hash_from_ip(struct bpf_trampoline *tr, void *ptr)

{

	unsigned long ip, addr = (unsigned long) ptr;

	struct ftrace_hash *hash;



	ip = ftrace_location(tr->ip);

	if (!ip)

		return NULL;

	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);

	if (!hash)

		return NULL;

	if (bpf_trampoline_use_jmp(tr->flags))

		addr = ftrace_jmp_set(addr);

	if (!add_ftrace_hash_entry_direct(hash, ip, addr)) {

		free_ftrace_hash(hash);

		return NULL;

	}

	return hash;

}



static int direct_ops_add(struct bpf_trampoline *tr, void *addr)

{

	struct ftrace_hash *hash = hash_from_ip(tr, addr);

	int err;



	if (!hash)

		return -ENOMEM;

	err = update_ftrace_direct_add(tr->fops, hash);

	free_ftrace_hash(hash);

	return err;

}



static int direct_ops_del(struct bpf_trampoline *tr, void *addr)

{

	struct ftrace_hash *hash = hash_from_ip(tr, addr);

	int err;



	if (!hash)

		return -ENOMEM;

	err = update_ftrace_direct_del(tr->fops, hash);

	free_ftrace_hash(hash);

	return err;

}



static int direct_ops_mod(struct bpf_trampoline *tr, void *addr, bool lock_direct_mutex)

{

	struct ftrace_hash *hash = hash_from_ip(tr, addr);

	int err;



	if (!hash)

		return -ENOMEM;

	err = update_ftrace_direct_mod(tr->fops, hash, lock_direct_mutex);

	free_ftrace_hash(hash);

	return err;

}

#else

/*

 * We allocate ftrace_ops object for each trampoline and it contains

 * call site specific for that trampoline.

 *

 * We use *_ftrace_direct api for attachment.

 */

static int direct_ops_alloc(struct bpf_trampoline *tr)

{

	tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);

	if (!tr->fops)

		return -ENOMEM;

	tr->fops->private = tr;

	tr->fops->ops_func = bpf_tramp_ftrace_ops_func;

	return 0;

}



static void direct_ops_free(struct bpf_trampoline *tr)

{

	if (!tr->fops)

		return;

	ftrace_free_filter(tr->fops);

	kfree(tr->fops);

}



static int direct_ops_add(struct bpf_trampoline *tr, void *ptr)

{

	unsigned long addr = (unsigned long) ptr;

	struct ftrace_ops *ops = tr->fops;

	int ret;



	if (bpf_trampoline_use_jmp(tr->flags))

		addr = ftrace_jmp_set(addr);



	ret = ftrace_set_filter_ip(ops, tr->ip, 0, 1);

	if (ret)

		return ret;

	return register_ftrace_direct(ops, addr);

}



static int direct_ops_del(struct bpf_trampoline *tr, void *addr)

{

	return unregister_ftrace_direct(tr->fops, (long)addr, false);

}



static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex)

{

	unsigned long addr = (unsigned long) ptr;

	struct ftrace_ops *ops = tr->fops;



	if (bpf_trampoline_use_jmp(tr->flags))

		addr = ftrace_jmp_set(addr);

	if (lock_direct_mutex)

		return modify_ftrace_direct(ops, addr);

	return modify_ftrace_direct_nolock(ops, addr);

}

#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */

#else

static void direct_ops_free(struct bpf_trampoline *tr) { }



static int direct_ops_alloc(struct bpf_trampoline *tr)

{

	return 0;

}



static int direct_ops_add(struct bpf_trampoline *tr, void *addr)

{

	return -ENODEV;

}



static int direct_ops_del(struct bpf_trampoline *tr, void *addr)

{

	return -ENODEV;

}



static int direct_ops_mod(struct bpf_trampoline *tr, void *ptr, bool lock_direct_mutex)

{

	return -ENODEV;

}

#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */



static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)

{

	struct bpf_trampoline *tr;
@@ -161,16 +345,11 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) 
	tr = kzalloc(sizeof(*tr), GFP_KERNEL);

	if (!tr)

		goto out;

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

	tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL);

	if (!tr->fops) {

	if (direct_ops_alloc(tr)) {

		kfree(tr);

		tr = NULL;

		goto out;

	}

	tr->fops->private = tr;

	tr->fops->ops_func = bpf_tramp_ftrace_ops_func;

#endif



	tr->key = key;

	tr->ip = ftrace_location(ip);
@@ -213,7 +392,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, 
	int ret;



	if (tr->func.ftrace_managed)

		ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);

		ret = direct_ops_del(tr, old_addr);

	else

		ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);
@@ -227,15 +406,7 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, 
	int ret;



	if (tr->func.ftrace_managed) {

		unsigned long addr = (unsigned long) new_addr;



		if (bpf_trampoline_use_jmp(tr->flags))

			addr = ftrace_jmp_set(addr);



		if (lock_direct_mutex)

			ret = modify_ftrace_direct(tr->fops, addr);

		else

			ret = modify_ftrace_direct_nolock(tr->fops, addr);

		ret = direct_ops_mod(tr, new_addr, lock_direct_mutex);

	} else {

		ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,

						   new_addr);
@@ -258,15 +429,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) 
	}



	if (tr->func.ftrace_managed) {

		unsigned long addr = (unsigned long) new_addr;



		if (bpf_trampoline_use_jmp(tr->flags))

			addr = ftrace_jmp_set(addr);



		ret = ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);

		if (ret)

			return ret;

		ret = register_ftrace_direct(tr->fops, addr);

		ret = direct_ops_add(tr, new_addr);

	} else {

		ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);

	}
@@ -947,10 +1110,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) 
	 */

	hlist_del(&tr->hlist_key);

	hlist_del(&tr->hlist_ip);

	if (tr->fops) {

		ftrace_free_filter(tr->fops);

		kfree(tr->fops);

	}

	direct_ops_free(tr);

	kfree(tr);

out:

	mutex_unlock(&trampoline_mutex);

kernel/trace/Kconfig

+3 −0
Original line number Diff line number Diff line
@@ -50,6 +50,9 @@ config HAVE_DYNAMIC_FTRACE_WITH_REGS 
config HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

	bool



config HAVE_SINGLE_FTRACE_DIRECT_OPS

	bool



config HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS

	bool

kernel/trace/ftrace.c

+6 −1
Original line number Diff line number Diff line
@@ -2631,8 +2631,13 @@ unsigned long ftrace_find_rec_direct(unsigned long ip) 
static void call_direct_funcs(unsigned long ip, unsigned long pip,

			      struct ftrace_ops *ops, struct ftrace_regs *fregs)

{

	unsigned long addr = READ_ONCE(ops->direct_call);

	unsigned long addr;



#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS

	addr = ftrace_find_rec_direct(ip);

#else

	addr = READ_ONCE(ops->direct_call);

#endif

	if (!addr)

		return;