Commit c9ebe201 authored by Tiezhu Yang's avatar Tiezhu Yang Committed by Huacai Chen
Browse files

LoongArch: BPF: Support up to 12 function arguments for trampoline 



Currently, LoongArch bpf trampoline supports up to 8 function arguments.
According to the statistics from commit 473e3150 ("bpf, x86: allow
function arguments up to 12 for TRACING"), there are over 200 functions
accept 9 to 12 arguments, so add 12 arguments support for trampoline.

With this patch, the following related testcases passed:

  sudo ./test_progs -a tracing_struct/struct_many_args
  sudo ./test_progs -a fentry_test/fentry_many_args
  sudo ./test_progs -a fexit_test/fexit_many_args

Acked-by: default avatarHengqi Chen <hengqi.chen@gmail.com>
Tested-by: default avatarHengqi Chen <hengqi.chen@gmail.com>
Signed-off-by: default avatarTiezhu Yang <yangtiezhu@loongson.cn>
Signed-off-by: default avatarHuacai Chen <chenhuacai@loongson.cn>
parent 0ef8b960

arch/loongarch/net/bpf_jit.c

+64 −35
Original line number Diff line number Diff line
@@ -1628,26 +1628,46 @@ int bpf_arch_text_invalidate(void *dst, size_t len) 
	return ret;

}



static void store_args(struct jit_ctx *ctx, int nregs, int args_off)

static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off)

{

	int i;



	for (i = 0; i < nregs; i++) {

	for (i = 0; i < nr_arg_slots; i++) {

		if (i < LOONGARCH_MAX_REG_ARGS)

			emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);

		else {

			/* Skip slots for T0 and FP of traced function */

			emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP,

				  16 + (i - LOONGARCH_MAX_REG_ARGS) * 8);

			emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off);

		}

		args_off -= 8;

	}

}



static void restore_args(struct jit_ctx *ctx, int nregs, int args_off)

static void restore_args(struct jit_ctx *ctx, int nr_reg_args, int args_off)

{

	int i;



	for (i = 0; i < nregs; i++) {

	for (i = 0; i < nr_reg_args; i++) {

		emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);

		args_off -= 8;

	}

}



static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, int stk_args_off)

{

	int i;



	for (i = 0; i < nr_stk_args; i++) {

		emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP,

			  -(args_off - LOONGARCH_MAX_REG_ARGS * 8));

		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -stk_args_off);

		args_off -= 8;

		stk_args_off -= 8;

	}

}



static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,

			   int args_off, int retval_off, int run_ctx_off, bool save_ret)

{
@@ -1763,8 +1783,8 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
					 void *func_addr, u32 flags)

{

	int i, ret, save_ret;

	int stack_size, nregs = m->nr_args;

	int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;

	int stack_size, args_off, stk_args_off, nr_arg_slots = 0;

	int retval_off, nregs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;

	bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;

	void *orig_call = func_addr;

	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
@@ -1797,11 +1817,17 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
	 * FP - sreg_off     [ callee saved reg  ]

	 *

	 * FP - tcc_ptr_off  [ tail_call_cnt_ptr ]

	 *

	 *                   [ stack_argN        ]

	 *                   [ ...               ]

	 * FP - stk_args_off [ stack_arg1        ] BPF_TRAMP_F_CALL_ORIG

	 */



	if (m->nr_args > MAX_BPF_FUNC_ARGS)

		return -ENOTSUPP;



	/* Extra registers for struct arguments */

	for (i = 0; i < m->nr_args; i++) {

		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) {

		/*

		 * The struct argument size is at most 16 bytes,

		 * enforced by the verifier. The struct argument
@@ -1809,13 +1835,9 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
		 * size is more than 8 bytes and no more than 16

		 * bytes.

		 */

			nregs += round_up(m->arg_size[i], 8) / 8 - 1;

		}

		nr_arg_slots += round_up(m->arg_size[i], 8) / 8;

	}



	if (nregs > LOONGARCH_MAX_REG_ARGS)

		return -ENOTSUPP;



	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))

		return -ENOTSUPP;
@@ -1829,7 +1851,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
	retval_off = stack_size;



	/* Room of trampoline frame to store args */

	stack_size += nregs * 8;

	stack_size += nr_arg_slots * 8;

	args_off = stack_size;



	/* Room of trampoline frame to store args number */
@@ -1855,8 +1877,14 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
		tcc_ptr_off = stack_size;

	}



	if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - LOONGARCH_MAX_REG_ARGS > 0))

		stack_size += (nr_arg_slots - LOONGARCH_MAX_REG_ARGS) * 8;



	stack_size = round_up(stack_size, 16);



	/* Room for args on stack must be at the top of stack */

	stk_args_off = stack_size;



	if (is_struct_ops) {

		/*

		 * For the trampoline called directly, just handle
@@ -1898,10 +1926,10 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
	}



	/* store arg regs count */

	move_imm(ctx, LOONGARCH_GPR_T1, nregs, false);

	move_imm(ctx, LOONGARCH_GPR_T1, nr_arg_slots, false);

	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nregs_off);



	store_args(ctx, nregs, args_off);

	store_args(ctx, nr_arg_slots, args_off);



	/* To traced function */

	/* Ftrace jump skips 2 NOP instructions */
@@ -1943,7 +1971,8 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
	}



	if (flags & BPF_TRAMP_F_CALL_ORIG) {

		restore_args(ctx, nregs, args_off);

		restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off);

		restore_stk_args(ctx, nr_arg_slots - LOONGARCH_MAX_REG_ARGS, args_off, stk_args_off);



		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)

			emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
@@ -1979,7 +2008,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i 
	}



	if (flags & BPF_TRAMP_F_RESTORE_REGS)

		restore_args(ctx, nregs, args_off);

		restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off);



	if (save_ret) {

		emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));