Commit 20a0bc10 authored by Jiri Olsa's avatar Jiri Olsa Committed by Alexei Starovoitov
Browse files

x86/fgraph,bpf: Fix stack ORC unwind from kprobe_multi return probe 



Currently we don't get stack trace via ORC unwinder on top of fgraph exit
handler. We can see that when generating stacktrace from kretprobe_multi
bpf program which is based on fprobe/fgraph.

The reason is that the ORC unwind code won't get pass the return_to_handler
callback installed by fgraph return probe machinery.

Solving this by creating stack frame in return_to_handler expected by
ftrace_graph_ret_addr function to recover original return address and
continue with the unwind.

Also updating the pt_regs data with cs/flags/rsp which are needed for
successful stack retrieval from ebpf bpf_get_stackid helper.
 - in get_perf_callchain we check user_mode(regs) so CS has to be set
 - in perf_callchain_kernel we call perf_hw_regs(regs), so EFLAGS/FIXED
    has to be unset

Acked-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Link: https://lore.kernel.org/r/20251104215405.168643-3-jolsa@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
Acked-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
parent 6d08340d

arch/x86/include/asm/ftrace.h

+5 −0
Original line number Diff line number Diff line
@@ -56,6 +56,11 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) 
	return &arch_ftrace_regs(fregs)->regs;

}



#define arch_ftrace_partial_regs(regs) do {	\

	regs->flags &= ~X86_EFLAGS_FIXED;	\

	regs->cs = __KERNEL_CS;			\

} while (0)



#define arch_ftrace_fill_perf_regs(fregs, _regs) do {	\

		(_regs)->ip = arch_ftrace_regs(fregs)->regs.ip;		\

		(_regs)->sp = arch_ftrace_regs(fregs)->regs.sp;		\

arch/x86/kernel/ftrace_64.S

+7 −1
Original line number Diff line number Diff line
@@ -354,12 +354,17 @@ SYM_CODE_START(return_to_handler) 
	UNWIND_HINT_UNDEFINED

	ANNOTATE_NOENDBR



	/* Restore return_to_handler value that got eaten by previous ret instruction. */

	subq $8, %rsp

	UNWIND_HINT_FUNC



	/* Save ftrace_regs for function exit context  */

	subq $(FRAME_SIZE), %rsp



	movq %rax, RAX(%rsp)

	movq %rdx, RDX(%rsp)

	movq %rbp, RBP(%rsp)

	movq %rsp, RSP(%rsp)

	movq %rsp, %rdi



	call ftrace_return_to_handler
@@ -368,7 +373,8 @@ SYM_CODE_START(return_to_handler) 
	movq RDX(%rsp), %rdx

	movq RAX(%rsp), %rax



	addq $(FRAME_SIZE), %rsp

	addq $(FRAME_SIZE) + 8, %rsp



	/*

	 * Jump back to the old return address. This cannot be JMP_NOSPEC rdi

	 * since IBT would demand that contain ENDBR, which simply isn't so for

include/linux/ftrace.h

+9 −1
Original line number Diff line number Diff line
@@ -193,6 +193,10 @@ static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs 
#if !defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) || \

	defined(CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS)



#ifndef arch_ftrace_partial_regs

#define arch_ftrace_partial_regs(regs) do {} while (0)

#endif



static __always_inline struct pt_regs *

ftrace_partial_regs(struct ftrace_regs *fregs, struct pt_regs *regs)

{
@@ -202,7 +206,11 @@ ftrace_partial_regs(struct ftrace_regs *fregs, struct pt_regs *regs) 
	 * Since arch_ftrace_get_regs() will check some members and may return

	 * NULL, we can not use it.

	 */

	return &arch_ftrace_regs(fregs)->regs;

	regs = &arch_ftrace_regs(fregs)->regs;



	/* Allow arch specific updates to regs. */

	arch_ftrace_partial_regs(regs);

	return regs;

}



#endif /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS || CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS */