Commit 3f2189e4 authored by Hou Tao's avatar Hou Tao Committed by Alexei Starovoitov
Browse files

bpf: Use pcpu_alloc_size() in bpf_mem_free{_rcu}() 



For bpf_global_percpu_ma, the pointer passed to bpf_mem_free_rcu() is
allocated by kmalloc() and its size is fixed (16-bytes on x86-64). So
no matter which cache allocates the dynamic per-cpu area, on x86-64
cache[2] will always be used to free the per-cpu area.

Fix the unbalance by checking whether the bpf memory allocator is
per-cpu or not and use pcpu_alloc_size() instead of ksize() to
find the correct cache for per-cpu free.

Signed-off-by: default avatarHou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231020133202.4043247-5-houtao@huaweicloud.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent baa8fdec

include/linux/bpf_mem_alloc.h

+1 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ struct bpf_mem_caches; 
struct bpf_mem_alloc {

	struct bpf_mem_caches __percpu *caches;

	struct bpf_mem_cache __percpu *cache;

	bool percpu;

	struct work_struct work;

};

kernel/bpf/memalloc.c

+14 −2
Original line number Diff line number Diff line
@@ -525,6 +525,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) 
	/* room for llist_node and per-cpu pointer */

	if (percpu)

		percpu_size = LLIST_NODE_SZ + sizeof(void *);

	ma->percpu = percpu;



	if (size) {

		pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
@@ -874,6 +875,17 @@ void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size) 
	return !ret ? NULL : ret + LLIST_NODE_SZ;

}



static notrace int bpf_mem_free_idx(void *ptr, bool percpu)

{

	size_t size;



	if (percpu)

		size = pcpu_alloc_size(*((void **)ptr));

	else

		size = ksize(ptr - LLIST_NODE_SZ);

	return bpf_mem_cache_idx(size);

}



void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)

{

	int idx;
@@ -881,7 +893,7 @@ void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr) 
	if (!ptr)

		return;



	idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));

	idx = bpf_mem_free_idx(ptr, ma->percpu);

	if (idx < 0)

		return;
@@ -895,7 +907,7 @@ void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr) 
	if (!ptr)

		return;



	idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));

	idx = bpf_mem_free_idx(ptr, ma->percpu);

	if (idx < 0)

		return;