Merge branch 'slab/for-6.11/buckets' into slab/for-next (436381ea) · Commits · git / linux-net

Documentation/core-api/memory-allocation.rst

+4 −2

Original line number	Diff line number	Diff line
		@@ -145,7 +145,9 @@ smaller than page size.

		The address of a chunk allocated with `kmalloc` is aligned to at least
		ARCH_KMALLOC_MINALIGN bytes. For sizes which are a power of two, the
		alignment is also guaranteed to be at least the respective size.
		alignment is also guaranteed to be at least the respective size. For other
		sizes, the alignment is guaranteed to be at least the largest power-of-two
		divisor of the size.

		Chunks allocated with kmalloc() can be resized with krealloc(). Similarly
		to kmalloc_array(): a helper for resizing arrays is provided in the form of

include/linux/mm.h

+3 −3

Original line number	Diff line number	Diff line
		@@ -1110,7 +1110,7 @@ static inline unsigned int compound_order(struct page *page)
		*
		* Return: The order of the folio.
		*/
		static inline unsigned int folio_order(struct folio *folio)
		static inline unsigned int folio_order(const struct folio *folio)
		{
		if (!folio_test_large(folio))
		return 0;
		@@ -2150,7 +2150,7 @@ static inline struct folio folio_next(struct folio folio)
		* it from being split. It is not necessary for the folio to be locked.
		* Return: The base-2 logarithm of the size of this folio.
		*/
		static inline unsigned int folio_shift(struct folio *folio)
		static inline unsigned int folio_shift(const struct folio *folio)
		{
		return PAGE_SHIFT + folio_order(folio);
		}
		@@ -2163,7 +2163,7 @@ static inline unsigned int folio_shift(struct folio *folio)
		* it from being split. It is not necessary for the folio to be locked.
		* Return: The number of bytes in this folio.
		*/
		static inline size_t folio_size(struct folio *folio)
		static inline size_t folio_size(const struct folio *folio)
		{
		return PAGE_SIZE << folio_order(folio);
		}

include/linux/poison.h

+2 −5

Original line number	Diff line number	Diff line
		@@ -38,11 +38,8 @@
		* Magic nums for obj red zoning.
		* Placed in the first word before and the first word after an obj.
		*/
		#define RED_INACTIVE 0x09F911029D74E35BULL /* when obj is inactive */
		#define RED_ACTIVE 0xD84156C5635688C0ULL /* when obj is active */

		#define SLUB_RED_INACTIVE 0xbb
		#define SLUB_RED_ACTIVE 0xcc
		#define SLUB_RED_INACTIVE 0xbb /* when obj is inactive */
		#define SLUB_RED_ACTIVE 0xcc /* when obj is active */

		/* ...and for poisoning */
		#define POISON_INUSE 0x5a /* for use-uninitialised poisoning */

include/linux/slab.h

+65 −32

Original line number	Diff line number	Diff line
		@@ -426,8 +426,9 @@ enum kmalloc_cache_type {
		NR_KMALLOC_TYPES
		};

		extern struct kmem_cache *
		kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1];
		typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1];

		extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES];

		/*
		* Define gfp bits that should not be set for KMALLOC_NORMAL.
		@@ -528,9 +529,6 @@ static_assert(PAGE_SHIFT <= 20);

		#include <linux/alloc_tag.h>

		void *__kmalloc_noprof(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1);
		#define __kmalloc(...) alloc_hooks(__kmalloc_noprof(__VA_ARGS__))

		/**
		* kmem_cache_alloc - Allocate an object
		* @cachep: The cache to allocate from.
		@@ -551,6 +549,10 @@ void kmem_cache_alloc_lru_noprof(struct kmem_cache s, struct list_lru *lru,

		void kmem_cache_free(struct kmem_cache s, void objp);

		kmem_buckets kmem_buckets_create(const char name, slab_flags_t flags,
		unsigned int useroffset, unsigned int usersize,
		void (ctor)(void ));

		/*
		* Bulk allocation and freeing operations. These are accelerated in an
		* allocator specific way to avoid taking locks repeatedly or building
		@@ -568,31 +570,49 @@ static __always_inline void kfree_bulk(size_t size, void **p)
		kmem_cache_free_bulk(NULL, size, p);
		}

		void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment
		__alloc_size(1);
		#define __kmalloc_node(...) alloc_hooks(__kmalloc_node_noprof(__VA_ARGS__))

		void kmem_cache_alloc_node_noprof(struct kmem_cache s, gfp_t flags,
		int node) __assume_slab_alignment __malloc;
		#define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))

		void kmalloc_trace_noprof(struct kmem_cache s, gfp_t flags, size_t size)
		__assume_kmalloc_alignment __alloc_size(3);
		/*
		* These macros allow declaring a kmem_buckets * parameter alongside size, which
		* can be compiled out with CONFIG_SLAB_BUCKETS=n so that a large number of call
		* sites don't have to pass NULL.
		*/
		#ifdef CONFIG_SLAB_BUCKETS
		#define DECL_BUCKET_PARAMS(_size, _b) size_t (_size), kmem_buckets *(_b)
		#define PASS_BUCKET_PARAMS(_size, _b) (_size), (_b)
		#define PASS_BUCKET_PARAM(_b) (_b)
		#else
		#define DECL_BUCKET_PARAMS(_size, _b) size_t (_size)
		#define PASS_BUCKET_PARAMS(_size, _b) (_size)
		#define PASS_BUCKET_PARAM(_b) NULL
		#endif

		void kmalloc_node_trace_noprof(struct kmem_cache s, gfp_t gfpflags,
		int node, size_t size) __assume_kmalloc_alignment
		__alloc_size(4);
		#define kmalloc_trace(...) alloc_hooks(kmalloc_trace_noprof(__VA_ARGS__))
		/*
		* The following functions are not to be used directly and are intended only
		* for internal use from kmalloc() and kmalloc_node()
		* with the exception of kunit tests
		*/

		#define kmalloc_node_trace(...) alloc_hooks(kmalloc_node_trace_noprof(__VA_ARGS__))
		void *__kmalloc_noprof(size_t size, gfp_t flags)
		__assume_kmalloc_alignment __alloc_size(1);

		void *kmalloc_large_noprof(size_t size, gfp_t flags) __assume_page_alignment
		__alloc_size(1);
		#define kmalloc_large(...) alloc_hooks(kmalloc_large_noprof(__VA_ARGS__))
		void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
		__assume_kmalloc_alignment __alloc_size(1);

		void __kmalloc_cache_noprof(struct kmem_cache s, gfp_t flags, size_t size)
		__assume_kmalloc_alignment __alloc_size(3);

		void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment
		__alloc_size(1);
		#define kmalloc_large_node(...) alloc_hooks(kmalloc_large_node_noprof(__VA_ARGS__))
		void __kmalloc_cache_node_noprof(struct kmem_cache s, gfp_t gfpflags,
		int node, size_t size)
		__assume_kmalloc_alignment __alloc_size(4);

		void *__kmalloc_large_noprof(size_t size, gfp_t flags)
		__assume_page_alignment __alloc_size(1);

		void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
		__assume_page_alignment __alloc_size(1);

		/**
		* kmalloc - allocate kernel memory
		@@ -604,7 +624,8 @@ void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_pag
		*
		* The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
		* bytes. For @size of power of two bytes, the alignment is also guaranteed
		* to be at least to the size.
		* to be at least to the size. For other sizes, the alignment is guaranteed to
		* be at least the largest power-of-two divisor of @size.
		*
		* The @flags argument may be one of the GFP flags defined at
		* include/linux/gfp_types.h and described at
		@@ -654,10 +675,10 @@ static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t f
		unsigned int index;

		if (size > KMALLOC_MAX_CACHE_SIZE)
		return kmalloc_large_noprof(size, flags);
		return __kmalloc_large_noprof(size, flags);

		index = kmalloc_index(size);
		return kmalloc_trace_noprof(
		return __kmalloc_cache_noprof(
		kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
		flags, size);
		}
		@@ -665,20 +686,26 @@ static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t f
		}
		#define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__))

		#define kmem_buckets_alloc(_b, _size, _flags) \
		alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))

		#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \
		alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_))

		static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)
		{
		if (__builtin_constant_p(size) && size) {
		unsigned int index;

		if (size > KMALLOC_MAX_CACHE_SIZE)
		return kmalloc_large_node_noprof(size, flags, node);
		return __kmalloc_large_node_noprof(size, flags, node);

		index = kmalloc_index(size);
		return kmalloc_node_trace_noprof(
		return __kmalloc_cache_node_noprof(
		kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
		flags, node, size);
		}
		return __kmalloc_node_noprof(size, flags, node);
		return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node);
		}
		#define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))

		@@ -729,8 +756,10 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(voi
		*/
		#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) \| __GFP_ZERO)

		void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, int node,
		void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node,
		unsigned long caller) __alloc_size(1);
		#define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
		__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller)
		#define kmalloc_node_track_caller(...) \
		alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))

		@@ -756,7 +785,7 @@ static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_
		return NULL;
		if (__builtin_constant_p(n) && __builtin_constant_p(size))
		return kmalloc_node_noprof(bytes, flags, node);
		return __kmalloc_node_noprof(bytes, flags, node);
		return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node);
		}
		#define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))

		@@ -780,7 +809,9 @@ static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
		#define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
		#define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)\|__GFP_ZERO, _node)

		extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_size(1);
		void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) __alloc_size(1);
		#define kvmalloc_node_noprof(size, flags, node) \
		__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node)
		#define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))

		#define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE)
		@@ -788,6 +819,8 @@ extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_si
		#define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)\|__GFP_ZERO)

		#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)\|__GFP_ZERO, _node)
		#define kmem_buckets_valloc(_b, _size, _flags) \
		alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))

		static inline __alloc_size(1, 2) void *
		kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)

ipc/msgutil.c

+12 −1

Original line number	Diff line number	Diff line
		@@ -42,6 +42,17 @@ struct msg_msgseg {
		#define DATALEN_MSG ((size_t)PAGE_SIZE-sizeof(struct msg_msg))
		#define DATALEN_SEG ((size_t)PAGE_SIZE-sizeof(struct msg_msgseg))

		static kmem_buckets *msg_buckets __ro_after_init;

		static int __init init_msg_buckets(void)
		{
		msg_buckets = kmem_buckets_create("msg_msg", SLAB_ACCOUNT,
		sizeof(struct msg_msg),
		DATALEN_MSG, NULL);

		return 0;
		}
		subsys_initcall(init_msg_buckets);

		static struct msg_msg *alloc_msg(size_t len)
		{
		@@ -50,7 +61,7 @@ static struct msg_msg *alloc_msg(size_t len)
		size_t alen;

		alen = min(len, DATALEN_MSG);
		msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL_ACCOUNT);
		msg = kmem_buckets_alloc(msg_buckets, sizeof(*msg) + alen, GFP_KERNEL);
		if (msg == NULL)
		return NULL;