Commit 26ef208c authored by Song Liu's avatar Song Liu Committed by Alexei Starovoitov
Browse files

bpf: Use arch_bpf_trampoline_size 



Instead of blindly allocating PAGE_SIZE for each trampoline, check the size
of the trampoline with arch_bpf_trampoline_size(). This size is saved in
bpf_tramp_image->size, and used for modmem charge/uncharge. The fallback
arch_alloc_bpf_trampoline() still allocates a whole page because we need to
use set_memory_* to protect the memory.

struct_ops trampoline still uses a whole page for multiple trampolines.

With this size check at caller (regular trampoline and struct_ops
trampoline), remove arch_bpf_trampoline_size() from
arch_prepare_bpf_trampoline() in archs.

Also, update bpf_image_ksym_add() to handle symbol of different sizes.

Signed-off-by: default avatarSong Liu <song@kernel.org>
Acked-by: default avatarIlya Leoshkevich <iii@linux.ibm.com>
Tested-by: Ilya Leoshkevich <iii@linux.ibm.com>  # on s390x
Acked-by: default avatarJiri Olsa <jolsa@kernel.org>
Acked-by: default avatarBjörn Töpel <bjorn@rivosinc.com>
Tested-by: Björn Töpel <bjorn@rivosinc.com> # on riscv
Link: https://lore.kernel.org/r/20231206224054.492250-7-song@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 96d1b7c0

arch/arm64/net/bpf_jit_comp.c

+0 −7
Original line number Diff line number Diff line
@@ -2079,13 +2079,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 
	if (nregs > 8)

		return -ENOTSUPP;



	ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr);

	if (ret < 0)

		return ret;



	if (ret > ((long)image_end - (long)image))

		return -EFBIG;



	jit_fill_hole(image, (unsigned int)(image_end - image));

	ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);

arch/riscv/net/bpf_jit_comp64.c

+0 −7
Original line number Diff line number Diff line
@@ -1052,13 +1052,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 
	int ret;

	struct rv_jit_context ctx;



	ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr);

	if (ret < 0)

		return ret;



	if (ret > (long)image_end - (long)image)

		return -EFBIG;



	ctx.ninsns = 0;

	/*

	 * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the

include/linux/bpf.h

+2 −1
Original line number Diff line number Diff line
@@ -1141,6 +1141,7 @@ enum bpf_tramp_prog_type { 


struct bpf_tramp_image {

	void *image;

	int size;

	struct bpf_ksym ksym;

	struct percpu_ref pcref;

	void *ip_after_call;
@@ -1325,7 +1326,7 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func 
void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,

				struct bpf_prog *to);

/* Called only from JIT-enabled code, so there's no need for stubs. */

void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym);

void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);

void bpf_image_ksym_del(struct bpf_ksym *ksym);

void bpf_ksym_add(struct bpf_ksym *ksym);

void bpf_ksym_del(struct bpf_ksym *ksym);

kernel/bpf/bpf_struct_ops.c

+7 −0
Original line number Diff line number Diff line
@@ -355,6 +355,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, 
				      void *image, void *image_end)

{

	u32 flags;

	int size;



	tlinks[BPF_TRAMP_FENTRY].links[0] = link;

	tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
@@ -362,6 +363,12 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, 
	 * and it must be used alone.

	 */

	flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0;



	size = arch_bpf_trampoline_size(model, flags, tlinks, NULL);

	if (size < 0)

		return size;

	if (size > (unsigned long)image_end - (unsigned long)image)

		return -E2BIG;

	return arch_prepare_bpf_trampoline(NULL, image, image_end,

					   model, flags, tlinks, NULL);

}

kernel/bpf/dispatcher.c

+1 −1
Original line number Diff line number Diff line
@@ -154,7 +154,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 
			d->image = NULL;

			goto out;

		}

		bpf_image_ksym_add(d->image, &d->ksym);

		bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym);

	}



	prev_num_progs = d->num_progs;