Commit 2cd82317 authored by Vitaly Wool's avatar Vitaly Wool Committed by Andrew Morton
Browse files

mm/slub: allow to set node and align in k[v]realloc 

Reimplement k[v]realloc_node() to be able to set node and alignment should
a user need to do so.  In order to do that while retaining the maximal
backward compatibility, add k[v]realloc_node_align() functions and
redefine the rest of API using these new ones.

While doing that, we also keep the number of _noprof variants to a
minimum, which implies some changes to the existing users of older _noprof
functions, that basically being bcachefs.

With that change we also provide the ability for the Rust part of the
kernel to set node and alignment in its K[v]xxx [re]allocations.

Link: https://lkml.kernel.org/r/20250806124147.1724658-1-vitaly.wool@konsulko.se


Signed-off-by: default avatarVitaly Wool <vitaly.wool@konsulko.se>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Danilo Krummrich <dakr@kernel.org>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Jann Horn <jannh@google.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 4c5d3365

fs/bcachefs/darray.c

+1 −1
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ int __bch2_darray_resize_noprof(darray_char *d, size_t element_size, size_t new_ 
			return -ENOMEM;



		void *data = likely(bytes < INT_MAX)

			? kvmalloc_noprof(bytes, gfp)

			? kvmalloc_node_align_noprof(bytes, 1, gfp, NUMA_NO_NODE)

			: vmalloc_noprof(bytes);

		if (!data)

			return -ENOMEM;

fs/bcachefs/util.h

+1 −1
Original line number Diff line number Diff line
@@ -61,7 +61,7 @@ static inline void *bch2_kvmalloc_noprof(size_t n, gfp_t flags) 
{

	void *p = unlikely(n >= INT_MAX)

		? vmalloc_noprof(n)

		: kvmalloc_noprof(n, flags & ~__GFP_ZERO);

		: kvmalloc_node_align_noprof(n, 1, flags & ~__GFP_ZERO, NUMA_NO_NODE);

	if (p && (flags & __GFP_ZERO))

		memset(p, 0, n);

	return p;

include/linux/bpfptr.h

+1 −1
Original line number Diff line number Diff line
@@ -67,7 +67,7 @@ static inline int copy_to_bpfptr_offset(bpfptr_t dst, size_t offset, 


static inline void *kvmemdup_bpfptr_noprof(bpfptr_t src, size_t len)

{

	void *p = kvmalloc_noprof(len, GFP_USER | __GFP_NOWARN);

	void *p = kvmalloc_node_align_noprof(len, 1, GFP_USER | __GFP_NOWARN, NUMA_NO_NODE);



	if (!p)

		return ERR_PTR(-ENOMEM);

include/linux/slab.h

+24 −15
Original line number Diff line number Diff line
@@ -465,9 +465,13 @@ int kmem_cache_shrink(struct kmem_cache *s); 
/*

 * Common kmalloc functions provided by all allocators

 */

void * __must_check krealloc_noprof(const void *objp, size_t new_size,

				    gfp_t flags) __realloc_size(2);

#define krealloc(...)				alloc_hooks(krealloc_noprof(__VA_ARGS__))

void * __must_check krealloc_node_align_noprof(const void *objp, size_t new_size,

					       unsigned long align,

					       gfp_t flags, int nid) __realloc_size(2);

#define krealloc_noprof(_o, _s, _f)	krealloc_node_align_noprof(_o, _s, 1, _f, NUMA_NO_NODE)

#define krealloc_node_align(...)	alloc_hooks(krealloc_node_align_noprof(__VA_ARGS__))

#define krealloc_node(_o, _s, _f, _n)	krealloc_node_align(_o, _s, 1, _f, _n)

#define krealloc(...)			krealloc_node(__VA_ARGS__, NUMA_NO_NODE)



void kfree(const void *objp);

void kfree_sensitive(const void *objp);
@@ -1041,18 +1045,20 @@ static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags) 
#define kzalloc(...)				alloc_hooks(kzalloc_noprof(__VA_ARGS__))

#define kzalloc_node(_size, _flags, _node)	kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)



void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) __alloc_size(1);

#define kvmalloc_node_noprof(size, flags, node)	\

	__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node)

#define kvmalloc_node(...)			alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))



#define kvmalloc(_size, _flags)			kvmalloc_node(_size, _flags, NUMA_NO_NODE)

#define kvmalloc_noprof(_size, _flags)		kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE)

void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,

			     gfp_t flags, int node) __alloc_size(1);

#define kvmalloc_node_align_noprof(_size, _align, _flags, _node)	\

	__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node)

#define kvmalloc_node_align(...)		\

	alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__))

#define kvmalloc_node(_s, _f, _n)		kvmalloc_node_align(_s, 1, _f, _n)

#define kvmalloc(...)				kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE)

#define kvzalloc(_size, _flags)			kvmalloc(_size, (_flags)|__GFP_ZERO)



#define kvzalloc_node(_size, _flags, _node)	kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)



#define kmem_buckets_valloc(_b, _size, _flags)	\

	alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))

	alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE))



static inline __alloc_size(1, 2) void *

kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
@@ -1062,7 +1068,7 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) 
	if (unlikely(check_mul_overflow(n, size, &bytes)))

		return NULL;



	return kvmalloc_node_noprof(bytes, flags, node);

	return kvmalloc_node_align_noprof(bytes, 1, flags, node);

}



#define kvmalloc_array_noprof(...)		kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
@@ -1073,9 +1079,12 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) 
#define kvcalloc_node(...)			alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))

#define kvcalloc(...)				alloc_hooks(kvcalloc_noprof(__VA_ARGS__))



void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)

		__realloc_size(2);

#define kvrealloc(...)				alloc_hooks(kvrealloc_noprof(__VA_ARGS__))

void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align,

				  gfp_t flags, int nid) __realloc_size(2);

#define kvrealloc_node_align(...)		\

	alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__))

#define kvrealloc_node(_p, _s, _f, _n)		kvrealloc_node_align(_p, _s, 1, _f, _n)

#define kvrealloc(...)				kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE)



extern void kvfree(const void *addr);

DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))

lib/rhashtable.c

+2 −2
Original line number Diff line number Diff line
@@ -184,8 +184,8 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, 
	static struct lock_class_key __key;



	tbl = alloc_hooks_tag(ht->alloc_tag,

			kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets),

					     gfp|__GFP_ZERO, NUMA_NO_NODE));

			kvmalloc_node_align_noprof(struct_size(tbl, buckets, nbuckets),

					     1, gfp|__GFP_ZERO, NUMA_NO_NODE));



	size = nbuckets;