Commit 56101b69 authored by Jiri Olsa's avatar Jiri Olsa Committed by Peter Zijlstra
Browse files

uprobes/x86: Add uprobe syscall to speed up uprobe 



Adding new uprobe syscall that calls uprobe handlers for given
'breakpoint' address.

The idea is that the 'breakpoint' address calls the user space
trampoline which executes the uprobe syscall.

The syscall handler reads the return address of the initial call
to retrieve the original 'breakpoint' address. With this address
we find the related uprobe object and call its consumers.

Adding the arch_uprobe_trampoline_mapping function that provides
uprobe trampoline mapping. This mapping is backed with one global
page initialized at __init time and shared by the all the mapping
instances.

We do not allow to execute uprobe syscall if the caller is not
from uprobe trampoline mapping.

The uprobe syscall ensures the consumer (bpf program) sees registers
values in the state before the trampoline was called.

Signed-off-by: default avatarJiri Olsa <jolsa@kernel.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Acked-by: default avatarOleg Nesterov <oleg@redhat.com>
Acked-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Link: https://lore.kernel.org/r/20250720112133.244369-10-jolsa@kernel.org
parent 91440ff4

arch/x86/entry/syscalls/syscall_64.tbl

+1 −0
Original line number Diff line number Diff line
@@ -345,6 +345,7 @@ 
333	common	io_pgetevents		sys_io_pgetevents

334	common	rseq			sys_rseq

335	common	uretprobe		sys_uretprobe

336	common	uprobe			sys_uprobe

# don't use numbers 387 through 423, add new calls after the last

# 'common' entry

424	common	pidfd_send_signal	sys_pidfd_send_signal

arch/x86/kernel/uprobes.c

+139 −0
Original line number Diff line number Diff line
@@ -752,6 +752,145 @@ void arch_uprobe_clear_state(struct mm_struct *mm) 
	hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node)

		destroy_uprobe_trampoline(tramp);

}



static bool __in_uprobe_trampoline(unsigned long ip)

{

	struct vm_area_struct *vma = vma_lookup(current->mm, ip);



	return vma && vma_is_special_mapping(vma, &tramp_mapping);

}



static bool in_uprobe_trampoline(unsigned long ip)

{

	struct mm_struct *mm = current->mm;

	bool found, retry = true;

	unsigned int seq;



	rcu_read_lock();

	if (mmap_lock_speculate_try_begin(mm, &seq)) {

		found = __in_uprobe_trampoline(ip);

		retry = mmap_lock_speculate_retry(mm, seq);

	}

	rcu_read_unlock();



	if (retry) {

		mmap_read_lock(mm);

		found = __in_uprobe_trampoline(ip);

		mmap_read_unlock(mm);

	}

	return found;

}



/*

 * See uprobe syscall trampoline; the call to the trampoline will push

 * the return address on the stack, the trampoline itself then pushes

 * cx, r11 and ax.

 */

struct uprobe_syscall_args {

	unsigned long ax;

	unsigned long r11;

	unsigned long cx;

	unsigned long retaddr;

};



SYSCALL_DEFINE0(uprobe)

{

	struct pt_regs *regs = task_pt_regs(current);

	struct uprobe_syscall_args args;

	unsigned long ip, sp;

	int err;



	/* Allow execution only from uprobe trampolines. */

	if (!in_uprobe_trampoline(regs->ip))

		goto sigill;



	err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args));

	if (err)

		goto sigill;



	ip = regs->ip;



	/*

	 * expose the "right" values of ax/r11/cx/ip/sp to uprobe_consumer/s, plus:

	 * - adjust ip to the probe address, call saved next instruction address

	 * - adjust sp to the probe's stack frame (check trampoline code)

	 */

	regs->ax  = args.ax;

	regs->r11 = args.r11;

	regs->cx  = args.cx;

	regs->ip  = args.retaddr - 5;

	regs->sp += sizeof(args);

	regs->orig_ax = -1;



	sp = regs->sp;



	handle_syscall_uprobe(regs, regs->ip);



	/*

	 * Some of the uprobe consumers has changed sp, we can do nothing,

	 * just return via iret.

	 */

	if (regs->sp != sp) {

		/* skip the trampoline call */

		if (args.retaddr - 5 == regs->ip)

			regs->ip += 5;

		return regs->ax;

	}



	regs->sp -= sizeof(args);



	/* for the case uprobe_consumer has changed ax/r11/cx */

	args.ax  = regs->ax;

	args.r11 = regs->r11;

	args.cx  = regs->cx;



	/* keep return address unless we are instructed otherwise */

	if (args.retaddr - 5 != regs->ip)

		args.retaddr = regs->ip;



	regs->ip = ip;



	err = copy_to_user((void __user *)regs->sp, &args, sizeof(args));

	if (err)

		goto sigill;



	/* ensure sysret, see do_syscall_64() */

	regs->r11 = regs->flags;

	regs->cx  = regs->ip;

	return 0;



sigill:

	force_sig(SIGILL);

	return -1;

}



asm (

	".pushsection .rodata\n"

	".balign " __stringify(PAGE_SIZE) "\n"

	"uprobe_trampoline_entry:\n"

	"push %rcx\n"

	"push %r11\n"

	"push %rax\n"

	"movq $" __stringify(__NR_uprobe) ", %rax\n"

	"syscall\n"

	"pop %rax\n"

	"pop %r11\n"

	"pop %rcx\n"

	"ret\n"

	".balign " __stringify(PAGE_SIZE) "\n"

	".popsection\n"

);



extern u8 uprobe_trampoline_entry[];



static int __init arch_uprobes_init(void)

{

	tramp_mapping_pages[0] = virt_to_page(uprobe_trampoline_entry);

	return 0;

}



late_initcall(arch_uprobes_init);



#else /* 32-bit: */

/*

 * No RIP-relative addressing on 32-bit

include/linux/syscalls.h

+2 −0
Original line number Diff line number Diff line
@@ -1005,6 +1005,8 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int on); 


asmlinkage long sys_uretprobe(void);



asmlinkage long sys_uprobe(void);



/* pciconfig: alpha, arm, arm64, ia64, sparc */

asmlinkage long sys_pciconfig_read(unsigned long bus, unsigned long dfn,

				unsigned long off, unsigned long len,

include/linux/uprobes.h

+1 −0
Original line number Diff line number Diff line
@@ -239,6 +239,7 @@ extern unsigned long uprobe_get_trampoline_vaddr(void); 
extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len);

extern void arch_uprobe_clear_state(struct mm_struct *mm);

extern void arch_uprobe_init_state(struct mm_struct *mm);

extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr);

#else /* !CONFIG_UPROBES */

struct uprobes_state {

};

kernel/events/uprobes.c

+17 −0
Original line number Diff line number Diff line
@@ -2771,6 +2771,23 @@ static void handle_swbp(struct pt_regs *regs) 
	rcu_read_unlock_trace();

}



void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr)

{

	struct uprobe *uprobe;

	int is_swbp;



	guard(rcu_tasks_trace)();



	uprobe = find_active_uprobe_rcu(bp_vaddr, &is_swbp);

	if (!uprobe)

		return;

	if (!get_utask())

		return;

	if (arch_uprobe_ignore(&uprobe->arch, regs))

		return;

	handler_chain(uprobe, regs);

}



/*

 * Perform required fix-ups and disable singlestep.

 * Allow pending signals to take effect.