Commit 402e44b3 authored by Menglong Dong's avatar Menglong Dong Committed by Alexei Starovoitov
Browse files

bpf: implement "jmp" mode for trampoline 



Implement the "jmp" mode for the bpf trampoline. For the ftrace_managed
case, we need only to set the FTRACE_OPS_FL_JMP on the tr->fops if "jmp"
is needed.

For the bpf poke case, we will check the origin poke type with the
"origin_flags", and current poke type with "tr->flags". The function
bpf_trampoline_update_fentry() is introduced to do the job.

The "jmp" mode will only be enabled with CONFIG_DYNAMIC_FTRACE_WITH_JMP
enabled and BPF_TRAMP_F_SHARE_IPMODIFY is not set. With
BPF_TRAMP_F_SHARE_IPMODIFY, we need to get the origin call ip from the
stack, so we can't use the "jmp" mode.

Signed-off-by: default avatarMenglong Dong <dongml2@chinatelecom.cn>
Acked-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Link: https://lore.kernel.org/r/20251118123639.688444-7-dongml2@chinatelecom.cn


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent ae4a3160

kernel/bpf/trampoline.c

+58 −17
Original line number Diff line number Diff line
@@ -175,24 +175,42 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 
	return tr;

}



static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)

static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flags,

					void *old_addr, void *new_addr)

{

	enum bpf_text_poke_type new_t = BPF_MOD_CALL, old_t = BPF_MOD_CALL;

	void *ip = tr->func.addr;



	if (!new_addr)

		new_t = BPF_MOD_NOP;

	else if (bpf_trampoline_use_jmp(tr->flags))

		new_t = BPF_MOD_JUMP;



	if (!old_addr)

		old_t = BPF_MOD_NOP;

	else if (bpf_trampoline_use_jmp(orig_flags))

		old_t = BPF_MOD_JUMP;



	return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr);

}



static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags,

			     void *old_addr)

{

	int ret;



	if (tr->func.ftrace_managed)

		ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);

	else

		ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, BPF_MOD_NOP,

					 old_addr, NULL);

		ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL);



	return ret;

}



static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr,

static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags,

			 void *old_addr, void *new_addr,

			 bool lock_direct_mutex)

{

	void *ip = tr->func.addr;

	int ret;



	if (tr->func.ftrace_managed) {
@@ -201,10 +219,8 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad 
		else

			ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr);

	} else {

		ret = bpf_arch_text_poke(ip,

					 old_addr ? BPF_MOD_CALL : BPF_MOD_NOP,

					 new_addr ? BPF_MOD_CALL : BPF_MOD_NOP,

					 old_addr, new_addr);

		ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr,

						   new_addr);

	}

	return ret;

}
@@ -229,8 +245,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) 
			return ret;

		ret = register_ftrace_direct(tr->fops, (long)new_addr);

	} else {

		ret = bpf_arch_text_poke(ip, BPF_MOD_NOP, BPF_MOD_CALL,

					 NULL, new_addr);

		ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr);

	}



	return ret;
@@ -416,7 +431,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut 
		return PTR_ERR(tlinks);



	if (total == 0) {

		err = unregister_fentry(tr, tr->cur_image->image);

		err = unregister_fentry(tr, orig_flags, tr->cur_image->image);

		bpf_tramp_image_put(tr->cur_image);

		tr->cur_image = NULL;

		goto out;
@@ -440,9 +455,20 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut 


#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

again:

	if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) &&

	    (tr->flags & BPF_TRAMP_F_CALL_ORIG))

		tr->flags |= BPF_TRAMP_F_ORIG_STACK;

	if (tr->flags & BPF_TRAMP_F_CALL_ORIG) {

		if (tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) {

			/* The BPF_TRAMP_F_SKIP_FRAME can be cleared in the

			 * first try, reset it in the second try.

			 */

			tr->flags |= BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SKIP_FRAME;

		} else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_JMP)) {

			/* Use "jmp" instead of "call" for the trampoline

			 * in the origin call case, and we don't need to

			 * skip the frame.

			 */

			tr->flags &= ~BPF_TRAMP_F_SKIP_FRAME;

		}

	}

#endif



	size = arch_bpf_trampoline_size(&tr->func.model, tr->flags,
@@ -473,10 +499,18 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut 
	if (err)

		goto out_free;



#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP

	if (bpf_trampoline_use_jmp(tr->flags))

		tr->fops->flags |= FTRACE_OPS_FL_JMP;

	else

		tr->fops->flags &= ~FTRACE_OPS_FL_JMP;

#endif



	WARN_ON(tr->cur_image && total == 0);

	if (tr->cur_image)

		/* progs already running at this address */

		err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex);

		err = modify_fentry(tr, orig_flags, tr->cur_image->image,

				    im->image, lock_direct_mutex);

	else

		/* first time registering */

		err = register_fentry(tr, im->image);
@@ -499,8 +533,15 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut 
	tr->cur_image = im;

out:

	/* If any error happens, restore previous flags */

	if (err)

	if (err) {

		tr->flags = orig_flags;

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP

		if (bpf_trampoline_use_jmp(tr->flags))

			tr->fops->flags |= FTRACE_OPS_FL_JMP;

		else

			tr->fops->flags &= ~FTRACE_OPS_FL_JMP;

#endif

	}

	kfree(tlinks);

	return err;