Commit 96d1b7c0 authored by Song Liu's avatar Song Liu Committed by Alexei Starovoitov
Browse files

bpf: Add arch_bpf_trampoline_size() 



This helper will be used to calculate the size of the trampoline before
allocating the memory.

arch_prepare_bpf_trampoline() for arm64 and riscv64 can use
arch_bpf_trampoline_size() to check the trampoline fits in the image.

OTOH, arch_prepare_bpf_trampoline() for s390 has to call the JIT process
twice, so it cannot use arch_bpf_trampoline_size().

Signed-off-by: default avatarSong Liu <song@kernel.org>
Acked-by: default avatarIlya Leoshkevich <iii@linux.ibm.com>
Tested-by: Ilya Leoshkevich <iii@linux.ibm.com>  # on s390x
Acked-by: default avatarJiri Olsa <jolsa@kernel.org>
Acked-by: default avatarBjörn Töpel <bjorn@rivosinc.com>
Tested-by: Björn Töpel <bjorn@rivosinc.com> # on riscv
Link: https://lore.kernel.org/r/20231206224054.492250-6-song@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 38b8b58a

arch/arm64/net/bpf_jit_comp.c

+41 −15
Original line number Diff line number Diff line
@@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 
	return ctx->idx;

}



int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,

				void *image_end, const struct btf_func_model *m,

				u32 flags, struct bpf_tramp_links *tlinks,

				void *func_addr)

static int btf_func_model_nregs(const struct btf_func_model *m)

{

	int i, ret;

	int nregs = m->nr_args;

	int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE;

	struct jit_ctx ctx = {

		.image = NULL,

		.idx = 0,

	};

	int i;



	/* extra registers needed for struct argument */

	for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
@@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 
			nregs += (m->arg_size[i] + 7) / 8 - 1;

	}



	return nregs;

}



int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

			     struct bpf_tramp_links *tlinks, void *func_addr)

{

	struct jit_ctx ctx = {

		.image = NULL,

		.idx = 0,

	};

	struct bpf_tramp_image im;

	int nregs, ret;



	nregs = btf_func_model_nregs(m);

	/* the first 8 registers are used for arguments */

	if (nregs > 8)

		return -ENOTSUPP;



	ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);

	ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags);

	if (ret < 0)

		return ret;



	if (ret > max_insns)

		return -EFBIG;

	return ret < 0 ? ret : ret * AARCH64_INSN_SIZE;

}



	ctx.image = image;

	ctx.idx = 0;

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,

				void *image_end, const struct btf_func_model *m,

				u32 flags, struct bpf_tramp_links *tlinks,

				void *func_addr)

{

	int ret, nregs;

	struct jit_ctx ctx = {

		.image = image,

		.idx = 0,

	};



	nregs = btf_func_model_nregs(m);

	/* the first 8 registers are used for arguments */

	if (nregs > 8)

		return -ENOTSUPP;



	ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr);

	if (ret < 0)

		return ret;



	if (ret > ((long)image_end - (long)image))

		return -EFBIG;



	jit_fill_hole(image, (unsigned int)(image_end - image));

	ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);

arch/riscv/net/bpf_jit_comp64.c

+17 −5
Original line number Diff line number Diff line
@@ -1029,6 +1029,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 
	return ret;

}



int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

			     struct bpf_tramp_links *tlinks, void *func_addr)

{

	struct bpf_tramp_image im;

	struct rv_jit_context ctx;

	int ret;



	ctx.ninsns = 0;

	ctx.insns = NULL;

	ctx.ro_insns = NULL;

	ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx);



	return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns);

}



int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,

				void *image_end, const struct btf_func_model *m,

				u32 flags, struct bpf_tramp_links *tlinks,
@@ -1037,14 +1052,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 
	int ret;

	struct rv_jit_context ctx;



	ctx.ninsns = 0;

	ctx.insns = NULL;

	ctx.ro_insns = NULL;

	ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);

	ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr);

	if (ret < 0)

		return ret;



	if (ninsns_rvoff(ret) > (long)image_end - (long)image)

	if (ret > (long)image_end - (long)image)

		return -EFBIG;



	ctx.ninsns = 0;

arch/s390/net/bpf_jit_comp.c

+34 −22
Original line number Diff line number Diff line
@@ -2637,6 +2637,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 
	return 0;

}



int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

			     struct bpf_tramp_links *tlinks, void *orig_call)

{

	struct bpf_tramp_image im;

	struct bpf_tramp_jit tjit;

	int ret;



	memset(&tjit, 0, sizeof(tjit));



	ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags,

					    tlinks, orig_call);



	return ret < 0 ? ret : tjit.common.prg;

}



int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,

				void *image_end, const struct btf_func_model *m,

				u32 flags, struct bpf_tramp_links *tlinks,
@@ -2644,19 +2659,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 
{

	struct bpf_tramp_jit tjit;

	int ret;

	int i;



	for (i = 0; i < 2; i++) {

		if (i == 0) {

	/* Compute offsets, check whether the code fits. */

	memset(&tjit, 0, sizeof(tjit));

		} else {

			/* Generate the code. */

			tjit.common.prg = 0;

			tjit.common.prg_buf = image;

		}

	ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags,

					    tlinks, func_addr);



	if (ret < 0)

		return ret;

	if (tjit.common.prg > (char *)image_end - (char *)image)
@@ -2665,9 +2673,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 
		 * BPF_MAX_TRAMP_LINKS.

		 */

		return -E2BIG;

	}



	return tjit.common.prg;

	tjit.common.prg = 0;

	tjit.common.prg_buf = image;

	ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags,

					    tlinks, func_addr);



	return ret < 0 ? ret : tjit.common.prg;

}



bool bpf_jit_supports_subprog_tailcalls(void)

arch/x86/net/bpf_jit_comp.c

+36 −4
Original line number Diff line number Diff line
@@ -2422,7 +2422,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, 
 * add rsp, 8                      // skip eth_type_trans's frame

 * ret                             // return to its caller

 */

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,

static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,

					 const struct btf_func_model *m, u32 flags,

					 struct bpf_tramp_links *tlinks,

					 void *func_addr)
@@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i 
	return ret;

}



int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,

				const struct btf_func_model *m, u32 flags,

				struct bpf_tramp_links *tlinks,

				void *func_addr)

{

	return __arch_prepare_bpf_trampoline(im, image, image_end, m, flags, tlinks, func_addr);

}



int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

			     struct bpf_tramp_links *tlinks, void *func_addr)

{

	struct bpf_tramp_image im;

	void *image;

	int ret;



	/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline().

	 * This will NOT cause fragmentation in direct map, as we do not

	 * call set_memory_*() on this buffer.

	 *

	 * We cannot use kvmalloc here, because we need image to be in

	 * module memory range.

	 */

	image = bpf_jit_alloc_exec(PAGE_SIZE);

	if (!image)

		return -ENOMEM;



	ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags,

					    tlinks, func_addr);

	bpf_jit_free_exec(image);

	return ret;

}



static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)

{

	u8 *jg_reloc, *prog = *pprog;

include/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -1106,6 +1106,8 @@ void *arch_alloc_bpf_trampoline(unsigned int size); 
void arch_free_bpf_trampoline(void *image, unsigned int size);

void arch_protect_bpf_trampoline(void *image, unsigned int size);

void arch_unprotect_bpf_trampoline(void *image, unsigned int size);

int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

			     struct bpf_tramp_links *tlinks, void *func_addr);



u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,

					     struct bpf_tramp_run_ctx *run_ctx);