Commit 7bd230a2 authored by Suren Baghdasaryan's avatar Suren Baghdasaryan Committed by Andrew Morton
Browse files

mm/slab: enable slab allocation tagging for kmalloc and friends 

Redefine kmalloc, krealloc, kzalloc, kcalloc, etc. to record allocations
and deallocations done by these functions.

[surenb@google.com: undo _noprof additions in the documentation]
  Link: https://lkml.kernel.org/r/20240326231453.1206227-7-surenb@google.com
[rdunlap@infradead.org: fix kcalloc() kernel-doc warnings]
  Link: https://lkml.kernel.org/r/20240327044649.9199-1-rdunlap@infradead.org
Link: https://lkml.kernel.org/r/20240321163705.3067592-26-surenb@google.com


Signed-off-by: default avatarSuren Baghdasaryan <surenb@google.com>
Co-developed-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: default avatarRandy Dunlap <rdunlap@infradead.org>
Reviewed-by: default avatarKees Cook <keescook@chromium.org>
Tested-by: default avatarKees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 53ed0af4

include/linux/fortify-string.h

+3 −2
Original line number Diff line number Diff line
@@ -725,9 +725,9 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) 
	return __real_memchr_inv(p, c, size);

}



extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup)

extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup_noprof)

								    __realloc_size(2);

__FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp)

__FORTIFY_INLINE void *kmemdup_noprof(const void * const POS0 p, size_t size, gfp_t gfp)

{

	const size_t p_size = __struct_size(p);
@@ -737,6 +737,7 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp 
		fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL);

	return __real_kmemdup(p, size, gfp);

}

#define kmemdup(...)	alloc_hooks(kmemdup_noprof(__VA_ARGS__))



/**

 * strcpy - Copy a string into another string buffer

include/linux/slab.h

+85 −84
Original line number Diff line number Diff line
@@ -271,7 +271,10 @@ int kmem_cache_shrink(struct kmem_cache *s); 
/*

 * Common kmalloc functions provided by all allocators

 */

void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2);

void * __must_check krealloc_noprof(const void *objp, size_t new_size,

				    gfp_t flags) __realloc_size(2);

#define krealloc(...)				alloc_hooks(krealloc_noprof(__VA_ARGS__))



void kfree(const void *objp);

void kfree_sensitive(const void *objp);

size_t __ksize(const void *objp);
@@ -523,7 +526,10 @@ static __always_inline unsigned int __kmalloc_index(size_t size, 
static_assert(PAGE_SHIFT <= 20);

#define kmalloc_index(s) __kmalloc_index(s, true)



void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1);

#include <linux/alloc_tag.h>



void *__kmalloc_noprof(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1);

#define __kmalloc(...)				alloc_hooks(__kmalloc_noprof(__VA_ARGS__))



/**

 * kmem_cache_alloc - Allocate an object
@@ -535,9 +541,14 @@ void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_siz 
 *

 * Return: pointer to the new object or %NULL in case of error

 */

void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) __assume_slab_alignment __malloc;

void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,

void *kmem_cache_alloc_noprof(struct kmem_cache *cachep,

			      gfp_t flags) __assume_slab_alignment __malloc;

#define kmem_cache_alloc(...)			alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__))



void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,

			    gfp_t gfpflags) __assume_slab_alignment __malloc;

#define kmem_cache_alloc_lru(...)	alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__))



void kmem_cache_free(struct kmem_cache *s, void *objp);



/*
@@ -548,29 +559,40 @@ void kmem_cache_free(struct kmem_cache *s, void *objp); 
 * Note that interrupts must be enabled when calling these functions.

 */

void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);

int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p);



int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);

#define kmem_cache_alloc_bulk(...)	alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))



static __always_inline void kfree_bulk(size_t size, void **p)

{

	kmem_cache_free_bulk(NULL, size, p);

}



void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment

void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment

							 __alloc_size(1);

void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment

									 __malloc;

#define __kmalloc_node(...)			alloc_hooks(__kmalloc_node_noprof(__VA_ARGS__))



void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags,

				   int node) __assume_slab_alignment __malloc;

#define kmem_cache_alloc_node(...)	alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))



void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)

void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t flags, size_t size)

		    __assume_kmalloc_alignment __alloc_size(3);



void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,

void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags,

		int node, size_t size) __assume_kmalloc_alignment

						__alloc_size(4);

void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment

#define kmalloc_trace(...)			alloc_hooks(kmalloc_trace_noprof(__VA_ARGS__))



#define kmalloc_node_trace(...)			alloc_hooks(kmalloc_node_trace_noprof(__VA_ARGS__))



void *kmalloc_large_noprof(size_t size, gfp_t flags) __assume_page_alignment

					      __alloc_size(1);

#define kmalloc_large(...)			alloc_hooks(kmalloc_large_noprof(__VA_ARGS__))



void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment

void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment

							     __alloc_size(1);

#define kmalloc_large_node(...)			alloc_hooks(kmalloc_large_node_noprof(__VA_ARGS__))



/**

 * kmalloc - allocate kernel memory
@@ -626,37 +648,39 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_align 
 *	Try really hard to succeed the allocation but fail

 *	eventually.

 */

static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)

static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags)

{

	if (__builtin_constant_p(size) && size) {

		unsigned int index;



		if (size > KMALLOC_MAX_CACHE_SIZE)

			return kmalloc_large(size, flags);

			return kmalloc_large_noprof(size, flags);



		index = kmalloc_index(size);

		return kmalloc_trace(

		return kmalloc_trace_noprof(

				kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],

				flags, size);

	}

	return __kmalloc(size, flags);

	return __kmalloc_noprof(size, flags);

}

#define kmalloc(...)				alloc_hooks(kmalloc_noprof(__VA_ARGS__))



static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)

static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)

{

	if (__builtin_constant_p(size) && size) {

		unsigned int index;



		if (size > KMALLOC_MAX_CACHE_SIZE)

			return kmalloc_large_node(size, flags, node);

			return kmalloc_large_node_noprof(size, flags, node);



		index = kmalloc_index(size);

		return kmalloc_node_trace(

		return kmalloc_node_trace_noprof(

				kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],

				flags, node, size);

	}

	return __kmalloc_node(size, flags, node);

	return __kmalloc_node_noprof(size, flags, node);

}

#define kmalloc_node(...)			alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))



/**

 * kmalloc_array - allocate memory for an array.
@@ -664,16 +688,17 @@ static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t fla 
 * @size: element size.

 * @flags: the type of memory to allocate (see kmalloc).

 */

static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_t flags)

static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags)

{

	size_t bytes;



	if (unlikely(check_mul_overflow(n, size, &bytes)))

		return NULL;

	if (__builtin_constant_p(n) && __builtin_constant_p(size))

		return kmalloc(bytes, flags);

	return __kmalloc(bytes, flags);

		return kmalloc_noprof(bytes, flags);

	return kmalloc_noprof(bytes, flags);

}

#define kmalloc_array(...)			alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))



/**

 * krealloc_array - reallocate memory for an array.
@@ -682,7 +707,7 @@ static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_ 
 * @new_size: new size of a single member of the array

 * @flags: the type of memory to allocate (see kmalloc)

 */

static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p,

static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p,

								       size_t new_n,

								       size_t new_size,

								       gfp_t flags)
@@ -692,8 +717,9 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, 
	if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))

		return NULL;



	return krealloc(p, bytes, flags);

	return krealloc_noprof(p, bytes, flags);

}

#define krealloc_array(...)			alloc_hooks(krealloc_array_noprof(__VA_ARGS__))



/**

 * kcalloc - allocate memory for an array. The memory is set to zero.
@@ -701,16 +727,12 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, 
 * @size: element size.

 * @flags: the type of memory to allocate (see kmalloc).

 */

static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flags)

{

	return kmalloc_array(n, size, flags | __GFP_ZERO);

}

#define kcalloc(n, size, flags)		kmalloc_array(n, size, (flags) | __GFP_ZERO)



void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,

void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, int node,

				  unsigned long caller) __alloc_size(1);

#define kmalloc_node_track_caller(size, flags, node) \

	__kmalloc_node_track_caller(size, flags, node, \

				    _RET_IP_)

#define kmalloc_node_track_caller(...)		\

	alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))



/*

 * kmalloc_track_caller is a special version of kmalloc that records the
@@ -720,11 +742,9 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, 
 * allocator where we care about the real place the memory allocation

 * request comes from.

 */

#define kmalloc_track_caller(size, flags) \

	__kmalloc_node_track_caller(size, flags, \

				    NUMA_NO_NODE, _RET_IP_)

#define kmalloc_track_caller(...)		kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE)



static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags,

static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,

							  int node)

{

	size_t bytes;
@@ -732,75 +752,56 @@ static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, 
	if (unlikely(check_mul_overflow(n, size, &bytes)))

		return NULL;

	if (__builtin_constant_p(n) && __builtin_constant_p(size))

		return kmalloc_node(bytes, flags, node);

	return __kmalloc_node(bytes, flags, node);

		return kmalloc_node_noprof(bytes, flags, node);

	return __kmalloc_node_noprof(bytes, flags, node);

}

#define kmalloc_array_node(...)			alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))



static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node)

{

	return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);

}

#define kcalloc_node(_n, _size, _flags, _node)	\

	kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node)



/*

 * Shortcuts

 */

static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags)

{

	return kmem_cache_alloc(k, flags | __GFP_ZERO);

}

#define kmem_cache_zalloc(_k, _flags)		kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)



/**

 * kzalloc - allocate memory. The memory is set to zero.

 * @size: how many bytes of memory are required.

 * @flags: the type of memory to allocate (see kmalloc).

 */

static inline __alloc_size(1) void *kzalloc(size_t size, gfp_t flags)

static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)

{

	return kmalloc(size, flags | __GFP_ZERO);

	return kmalloc_noprof(size, flags | __GFP_ZERO);

}

#define kzalloc(...)				alloc_hooks(kzalloc_noprof(__VA_ARGS__))

#define kzalloc_node(_size, _flags, _node)	kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)



/**

 * kzalloc_node - allocate zeroed memory from a particular memory node.

 * @size: how many bytes of memory are required.

 * @flags: the type of memory to allocate (see kmalloc).

 * @node: memory node from which to allocate

 */

static inline __alloc_size(1) void *kzalloc_node(size_t size, gfp_t flags, int node)

{

	return kmalloc_node(size, flags | __GFP_ZERO, node);

}

extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_size(1);

#define kvmalloc_node(...)			alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))



extern void *kvmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1);

static inline __alloc_size(1) void *kvmalloc(size_t size, gfp_t flags)

{

	return kvmalloc_node(size, flags, NUMA_NO_NODE);

}

static inline __alloc_size(1) void *kvzalloc_node(size_t size, gfp_t flags, int node)

{

	return kvmalloc_node(size, flags | __GFP_ZERO, node);

}

static inline __alloc_size(1) void *kvzalloc(size_t size, gfp_t flags)

{

	return kvmalloc(size, flags | __GFP_ZERO);

}

#define kvmalloc(_size, _flags)			kvmalloc_node(_size, _flags, NUMA_NO_NODE)

#define kvzalloc(_size, _flags)			kvmalloc(_size, _flags|__GFP_ZERO)



#define kvzalloc_node(_size, _flags, _node)	kvmalloc_node(_size, _flags|__GFP_ZERO, _node)



static inline __alloc_size(1, 2) void *kvmalloc_array(size_t n, size_t size, gfp_t flags)

static inline __alloc_size(1, 2) void *kvmalloc_array_noprof(size_t n, size_t size, gfp_t flags)

{

	size_t bytes;



	if (unlikely(check_mul_overflow(n, size, &bytes)))

		return NULL;



	return kvmalloc(bytes, flags);

	return kvmalloc_node_noprof(bytes, flags, NUMA_NO_NODE);

}



static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags)

{

	return kvmalloc_array(n, size, flags | __GFP_ZERO);

}

#define kvmalloc_array(...)			alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))

#define kvcalloc(_n, _size, _flags)		kvmalloc_array(_n, _size, _flags|__GFP_ZERO)



extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)

extern void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags)

		      __realloc_size(3);

#define kvrealloc(...)				alloc_hooks(kvrealloc_noprof(__VA_ARGS__))



extern void kvfree(const void *addr);

DEFINE_FREE(kvfree, void *, if (_T) kvfree(_T))

include/linux/string.h

+3 −1
Original line number Diff line number Diff line
@@ -282,7 +282,9 @@ extern void kfree_const(const void *x); 
extern char *kstrdup(const char *s, gfp_t gfp) __malloc;

extern const char *kstrdup_const(const char *s, gfp_t gfp);

extern char *kstrndup(const char *s, size_t len, gfp_t gfp);

extern void *kmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2);

extern void *kmemdup_noprof(const void *src, size_t len, gfp_t gfp) __realloc_size(2);

#define kmemdup(...)	alloc_hooks(kmemdup_noprof(__VA_ARGS__))



extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2);

extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp);

extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp);

mm/slab_common.c

+3 −3
Original line number Diff line number Diff line
@@ -1189,7 +1189,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t flags) 
		return (void *)p;

	}



	ret = kmalloc_track_caller(new_size, flags);

	ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_);

	if (ret && p) {

		/* Disable KASAN checks as the object's redzone is accessed. */

		kasan_disable_current();
@@ -1213,7 +1213,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t flags) 
 *

 * Return: pointer to the allocated memory or %NULL in case of error

 */

void *krealloc(const void *p, size_t new_size, gfp_t flags)

void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)

{

	void *ret;
@@ -1228,7 +1228,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags) 


	return ret;

}

EXPORT_SYMBOL(krealloc);

EXPORT_SYMBOL(krealloc_noprof);



/**

 * kfree_sensitive - Clear sensitive information in memory before freeing

mm/slub.c

+25 −25
Original line number Diff line number Diff line
@@ -4002,7 +4002,7 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list 
	return object;

}



void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)

void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)

{

	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,

				    s->object_size);
@@ -4011,9 +4011,9 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) 


	return ret;

}

EXPORT_SYMBOL(kmem_cache_alloc);

EXPORT_SYMBOL(kmem_cache_alloc_noprof);



void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,

void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,

			   gfp_t gfpflags)

{

	void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
@@ -4023,7 +4023,7 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, 


	return ret;

}

EXPORT_SYMBOL(kmem_cache_alloc_lru);

EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof);



/**

 * kmem_cache_alloc_node - Allocate an object on the specified node
@@ -4038,7 +4038,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_lru); 
 *

 * Return: pointer to the new object or %NULL in case of error

 */

void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)

void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)

{

	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
@@ -4046,7 +4046,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) 


	return ret;

}

EXPORT_SYMBOL(kmem_cache_alloc_node);

EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);



/*

 * To avoid unnecessary overhead, we pass through large allocation requests
@@ -4063,7 +4063,7 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) 
		flags = kmalloc_fix_flags(flags);



	flags |= __GFP_COMP;

	folio = (struct folio *)alloc_pages_node(node, flags, order);

	folio = (struct folio *)alloc_pages_node_noprof(node, flags, order);

	if (folio) {

		ptr = folio_address(folio);

		lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
@@ -4078,7 +4078,7 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) 
	return ptr;

}



void *kmalloc_large(size_t size, gfp_t flags)

void *kmalloc_large_noprof(size_t size, gfp_t flags)

{

	void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
@@ -4086,9 +4086,9 @@ void *kmalloc_large(size_t size, gfp_t flags) 
		      flags, NUMA_NO_NODE);

	return ret;

}

EXPORT_SYMBOL(kmalloc_large);

EXPORT_SYMBOL(kmalloc_large_noprof);



void *kmalloc_large_node(size_t size, gfp_t flags, int node)

void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)

{

	void *ret = __kmalloc_large_node(size, flags, node);
@@ -4096,7 +4096,7 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) 
		      flags, node);

	return ret;

}

EXPORT_SYMBOL(kmalloc_large_node);

EXPORT_SYMBOL(kmalloc_large_node_noprof);



static __always_inline

void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
@@ -4123,26 +4123,26 @@ void *__do_kmalloc_node(size_t size, gfp_t flags, int node, 
	return ret;

}



void *__kmalloc_node(size_t size, gfp_t flags, int node)

void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node)

{

	return __do_kmalloc_node(size, flags, node, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc_node);

EXPORT_SYMBOL(__kmalloc_node_noprof);



void *__kmalloc(size_t size, gfp_t flags)

void *__kmalloc_noprof(size_t size, gfp_t flags)

{

	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc);

EXPORT_SYMBOL(__kmalloc_noprof);



void *__kmalloc_node_track_caller(size_t size, gfp_t flags,

void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags,

				       int node, unsigned long caller)

{

	return __do_kmalloc_node(size, flags, node, caller);

}

EXPORT_SYMBOL(__kmalloc_node_track_caller);

EXPORT_SYMBOL(kmalloc_node_track_caller_noprof);



void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)

void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)

{

	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,

					    _RET_IP_, size);
@@ -4152,9 +4152,9 @@ void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) 
	ret = kasan_kmalloc(s, ret, size, gfpflags);

	return ret;

}

EXPORT_SYMBOL(kmalloc_trace);

EXPORT_SYMBOL(kmalloc_trace_noprof);



void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,

void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags,

			 int node, size_t size)

{

	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
@@ -4164,7 +4164,7 @@ void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, 
	ret = kasan_kmalloc(s, ret, size, gfpflags);

	return ret;

}

EXPORT_SYMBOL(kmalloc_node_trace);

EXPORT_SYMBOL(kmalloc_node_trace_noprof);



static noinline void free_to_partial_list(

	struct kmem_cache *s, struct slab *slab,
@@ -4769,7 +4769,7 @@ static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, 
#endif /* CONFIG_SLUB_TINY */



/* Note that interrupts must be enabled when calling this function. */

int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,

int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,

				 void **p)

{

	int i;
@@ -4798,7 +4798,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 


	return i;

}

EXPORT_SYMBOL(kmem_cache_alloc_bulk);

EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);





/*