Commit 0c3772a8 authored by Menglong Dong's avatar Menglong Dong Committed by Alexei Starovoitov
Browse files

x86/ftrace: Implement DYNAMIC_FTRACE_WITH_JMP 



Implement the DYNAMIC_FTRACE_WITH_JMP for x86_64. In ftrace_call_replace,
we will use JMP32_INSN_OPCODE instead of CALL_INSN_OPCODE if the address
should use "jmp".

Meanwhile, adjust the direct call in the ftrace_regs_caller. The RSB is
balanced in the "jmp" mode. Take the function "foo" for example:

 original_caller:
 call foo -> foo:
         call fentry -> fentry:
                 [do ftrace callbacks ]
                 move tramp_addr to stack
                 RET -> tramp_addr
                         tramp_addr:
                         [..]
                         call foo_body -> foo_body:
                                 [..]
                                 RET -> back to tramp_addr
                         [..]
                         RET -> back to original_caller

Signed-off-by: default avatarMenglong Dong <dongml2@chinatelecom.cn>
Acked-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20251118123639.688444-3-dongml2@chinatelecom.cn


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 25e4e356

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -230,6 +230,7 @@ config X86 
	select HAVE_DYNAMIC_FTRACE_WITH_ARGS	if X86_64

	select HAVE_FTRACE_REGS_HAVING_PT_REGS	if X86_64

	select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

	select HAVE_DYNAMIC_FTRACE_WITH_JMP	if X86_64

	select HAVE_SAMPLE_FTRACE_DIRECT	if X86_64

	select HAVE_SAMPLE_FTRACE_DIRECT_MULTI	if X86_64

	select HAVE_EBPF_JIT

arch/x86/kernel/ftrace.c

+6 −1
Original line number Diff line number Diff line
@@ -74,8 +74,13 @@ static const char *ftrace_call_replace(unsigned long ip, unsigned long addr) 
	 * No need to translate into a callthunk. The trampoline does

	 * the depth accounting itself.

	 */

	if (ftrace_is_jmp(addr)) {

		addr = ftrace_jmp_get(addr);

		return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr);

	} else {

		return text_gen_insn(CALL_INSN_OPCODE, (void *)ip, (void *)addr);

	}

}



static int ftrace_verify_code(unsigned long ip, const char *old_code)

{

arch/x86/kernel/ftrace_64.S

+11 −1
Original line number Diff line number Diff line
@@ -285,8 +285,18 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) 
	ANNOTATE_NOENDBR

	RET



1:

	testb	$1, %al

	jz	2f

	andq $0xfffffffffffffffe, %rax

	movq %rax, MCOUNT_REG_SIZE+8(%rsp)

	restore_mcount_regs

	/* Restore flags */

	popfq

	RET



	/* Swap the flags with orig_rax */

1:	movq MCOUNT_REG_SIZE(%rsp), %rdi

2:	movq MCOUNT_REG_SIZE(%rsp), %rdi

	movq %rdi, MCOUNT_REG_SIZE-8(%rsp)

	movq %rax, MCOUNT_REG_SIZE(%rsp)