Commit 77620d12 authored by Eduard Zingerman's avatar Eduard Zingerman Committed by Alexei Starovoitov
Browse files

bpf: use realloc in bpf_patch_insn_data 

Avoid excessive vzalloc/vfree calls when patching instructions in
do_misc_fixups(). bpf_patch_insn_data() uses vzalloc to allocate new
memory for env->insn_aux_data for each patch as follows:

  struct bpf_prog *bpf_patch_insn_data(env, ...)
  {
    ...
    new_data = vzalloc(... O(program size) ...);
    ...
    adjust_insn_aux_data(env, new_data, ...);
    ...
  }

  void adjust_insn_aux_data(env, new_data, ...)
  {
    ...
    memcpy(new_data, env->insn_aux_data);
    vfree(env->insn_aux_data);
    env->insn_aux_data = new_data;
    ...
  }

The vzalloc/vfree pair is hot in perf report collected for e.g.
pyperf180 test case. It can be replaced with a call to vrealloc in
order to reduce the number of actual memory allocations.

This is a stop-gap solution, as bpf_patch_insn_data is still hot in
the profile. More comprehansive solutions had been discussed before
e.g. as in [1].

[1] https://lore.kernel.org/bpf/CAEf4BzY_E8MSL4mD0UPuuiDcbJhh9e2xQo2=5w+ppRWWiYSGvQ@mail.gmail.com/



Suggested-by: default avatarAlexei Starovoitov <alexei.starovoitov@gmail.com>
Signed-off-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Acked-by: default avatarKumar Kartikeya Dwivedi <memxor@gmail.com>
Tested-by: default avatarAnton Protopopov <a.s.protopopov@gmail.com>
Link: https://lore.kernel.org/r/20250807010205.3210608-3-eddyz87@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent cb070a81

kernel/bpf/verifier.c

+15 −14
Original line number Diff line number Diff line
@@ -20699,12 +20699,11 @@ static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 
 * [0, off) and [off, end) to new locations, so the patched range stays zero

 */

static void adjust_insn_aux_data(struct bpf_verifier_env *env,

				 struct bpf_insn_aux_data *new_data,

				 struct bpf_prog *new_prog, u32 off, u32 cnt)

{

	struct bpf_insn_aux_data *old_data = env->insn_aux_data;

	struct bpf_insn_aux_data *data = env->insn_aux_data;

	struct bpf_insn *insn = new_prog->insnsi;

	u32 old_seen = old_data[off].seen;

	u32 old_seen = data[off].seen;

	u32 prog_len;

	int i;








@@ -20712,22 +20711,20 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,







	 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the









	 * original insn at old prog.









	 */









	old_data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);









	data[off].zext_dst = insn_has_def32(insn + off + cnt - 1);

















	if (cnt == 1)









		return;









	prog_len = new_prog->len;

















	memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);









	memcpy(new_data + off + cnt - 1, old_data + off,









	memmove(data + off + cnt - 1, data + off,









		sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));









	memset(data + off, 0, sizeof(struct bpf_insn_aux_data) * (cnt - 1));









	for (i = off; i < off + cnt - 1; i++) {









		/* Expand insni[off]'s seen count to the patched range. */









		new_data[i].seen = old_seen;









		new_data[i].zext_dst = insn_has_def32(insn + i);









		data[i].seen = old_seen;









		data[i].zext_dst = insn_has_def32(insn + i);









	}









	env->insn_aux_data = new_data;









	vfree(old_data);









}

















static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)








@@ -20765,10 +20762,14 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of







	struct bpf_insn_aux_data *new_data = NULL;

















	if (len > 1) {









		new_data = vzalloc(array_size(env->prog->len + len - 1,









					      sizeof(struct bpf_insn_aux_data)));









		new_data = vrealloc(env->insn_aux_data,









				    array_size(env->prog->len + len - 1,









					       sizeof(struct bpf_insn_aux_data)),









				    GFP_KERNEL_ACCOUNT | __GFP_ZERO);









		if (!new_data)









			return NULL;

















		env->insn_aux_data = new_data;









	}

















	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);








@@ -20780,7 +20781,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of







		vfree(new_data);









		return NULL;









	}









	adjust_insn_aux_data(env, new_data, new_prog, off, len);









	adjust_insn_aux_data(env, new_prog, off, len);









	adjust_subprog_starts(env, off, len);









	adjust_poke_descs(new_prog, off, len);









	return new_prog;