Commit 25e4e356 authored by Menglong Dong's avatar Menglong Dong Committed by Alexei Starovoitov
Browse files

ftrace: Introduce FTRACE_OPS_FL_JMP 



For now, the "nop" will be replaced with a "call" instruction when a
function is hooked by the ftrace. However, sometimes the "call" can break
the RSB and introduce extra overhead. Therefore, introduce the flag
FTRACE_OPS_FL_JMP, which indicate that the ftrace_ops should be called
with a "jmp" instead of "call". For now, it is only used by the direct
call case.

When a direct ftrace_ops is marked with FTRACE_OPS_FL_JMP, the last bit of
the ops->direct_call will be set to 1. Therefore, we can tell if we should
use "jmp" for the callback in ftrace_call_replace().

Signed-off-by: default avatarMenglong Dong <dongml2@chinatelecom.cn>
Acked-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20251118123639.688444-2-dongml2@chinatelecom.cn


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent fad80400

include/linux/ftrace.h

+33 −0
Original line number Diff line number Diff line
@@ -359,6 +359,7 @@ enum { 
	FTRACE_OPS_FL_DIRECT			= BIT(17),

	FTRACE_OPS_FL_SUBOP			= BIT(18),

	FTRACE_OPS_FL_GRAPH			= BIT(19),

	FTRACE_OPS_FL_JMP			= BIT(20),

};



#ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
@@ -577,6 +578,38 @@ static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, 
						 unsigned long addr) { }

#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */



#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP

static inline bool ftrace_is_jmp(unsigned long addr)

{

	return addr & 1;

}



static inline unsigned long ftrace_jmp_set(unsigned long addr)

{

	return addr | 1UL;

}



static inline unsigned long ftrace_jmp_get(unsigned long addr)

{

	return addr & ~1UL;

}

#else

static inline bool ftrace_is_jmp(unsigned long addr)

{

	return false;

}



static inline unsigned long ftrace_jmp_set(unsigned long addr)

{

	return addr;

}



static inline unsigned long ftrace_jmp_get(unsigned long addr)

{

	return addr;

}

#endif /* CONFIG_DYNAMIC_FTRACE_WITH_JMP */



#ifdef CONFIG_STACK_TRACER



int stack_trace_sysctl(const struct ctl_table *table, int write, void *buffer,

kernel/trace/Kconfig

+12 −0
Original line number Diff line number Diff line
@@ -80,6 +80,12 @@ config HAVE_DYNAMIC_FTRACE_NO_PATCHABLE 
	  If the architecture generates __patchable_function_entries sections

	  but does not want them included in the ftrace locations.



config HAVE_DYNAMIC_FTRACE_WITH_JMP

	bool

	help

	  If the architecture supports to replace the __fentry__ with a

	  "jmp" instruction.



config HAVE_SYSCALL_TRACEPOINTS

	bool

	help
@@ -330,6 +336,12 @@ config DYNAMIC_FTRACE_WITH_ARGS 
	depends on DYNAMIC_FTRACE

	depends on HAVE_DYNAMIC_FTRACE_WITH_ARGS



config DYNAMIC_FTRACE_WITH_JMP

	def_bool y

	depends on DYNAMIC_FTRACE

	depends on DYNAMIC_FTRACE_WITH_DIRECT_CALLS

	depends on HAVE_DYNAMIC_FTRACE_WITH_JMP



config FPROBE

	bool "Kernel Function Probe (fprobe)"

	depends on HAVE_FUNCTION_GRAPH_FREGS && HAVE_FTRACE_GRAPH_FUNC

kernel/trace/ftrace.c

+16 −1
Original line number Diff line number Diff line
@@ -5951,7 +5951,8 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long 
	for (i = 0; i < size; i++) {

		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {

			del = __ftrace_lookup_ip(direct_functions, entry->ip);

			if (del && del->direct == addr) {

			if (del && ftrace_jmp_get(del->direct) ==

				   ftrace_jmp_get(addr)) {

				remove_hash_entry(direct_functions, del);

				kfree(del);

			}
@@ -6016,8 +6017,15 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) 
	if (ftrace_hash_empty(hash))

		return -EINVAL;



	/* This is a "raw" address, and this should never happen. */

	if (WARN_ON_ONCE(ftrace_is_jmp(addr)))

		return -EINVAL;



	mutex_lock(&direct_mutex);



	if (ops->flags & FTRACE_OPS_FL_JMP)

		addr = ftrace_jmp_set(addr);



	/* Make sure requested entries are not already registered.. */

	size = 1 << hash->size_bits;

	for (i = 0; i < size; i++) {
@@ -6138,6 +6146,13 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) 


	lockdep_assert_held_once(&direct_mutex);



	/* This is a "raw" address, and this should never happen. */

	if (WARN_ON_ONCE(ftrace_is_jmp(addr)))

		return -EINVAL;



	if (ops->flags & FTRACE_OPS_FL_JMP)

		addr = ftrace_jmp_set(addr);



	/* Enable the tmp_ops to have the same functions as the direct ops */

	ftrace_ops_init(&tmp_ops);

	tmp_ops.func_hash = ops->func_hash;