Commit 4c0a17e2 authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

slab: prevent recursive kmalloc() in alloc_empty_sheaf() 

We want to expand usage of sheaves to all non-boot caches, including
kmalloc caches. Since sheaves themselves are also allocated by
kmalloc(), we need to prevent excessive or infinite recursion -
depending on sheaf size, the sheaf can be allocated from smaller, same
or larger kmalloc size bucket, there's no particular constraint.

This is similar to allocating the objext arrays so let's just reuse the
existing mechanisms for those. __GFP_NO_OBJ_EXT in alloc_empty_sheaf()
will prevent a nested kmalloc() from allocating a sheaf itself - it will
either have sheaves already, or fallback to a non-sheaf-cached
allocation (so bootstrap of sheaves in a kmalloc cache that allocates
sheaves from its own size bucket is possible). Additionally, reuse
OBJCGS_CLEAR_MASK to clear unwanted gfp flags from the nested
allocation.

Link: https://patch.msgid.link/20251105-sheaves-cleanups-v1-5-b8218e1ac7ef@suse.cz


Reviewed-by: default avatarHarry Yoo <harry.yoo@oracle.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 31e0886f

include/linux/gfp_types.h

+0 −6
Original line number Diff line number Diff line
@@ -55,9 +55,7 @@ enum { 
#ifdef CONFIG_LOCKDEP

	___GFP_NOLOCKDEP_BIT,

#endif

#ifdef CONFIG_SLAB_OBJ_EXT

	___GFP_NO_OBJ_EXT_BIT,

#endif

	___GFP_LAST_BIT

};
@@ -98,11 +96,7 @@ enum { 
#else

#define ___GFP_NOLOCKDEP	0

#endif

#ifdef CONFIG_SLAB_OBJ_EXT

#define ___GFP_NO_OBJ_EXT       BIT(___GFP_NO_OBJ_EXT_BIT)

#else

#define ___GFP_NO_OBJ_EXT       0

#endif



/*

 * Physical address zone modifiers (see linux/mmzone.h - low four bits)

mm/slub.c

+26 −10
Original line number Diff line number Diff line
@@ -2031,6 +2031,14 @@ static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 
}

#endif /* CONFIG_SLUB_DEBUG */



/*

 * The allocated objcg pointers array is not accounted directly.

 * Moreover, it should not come from DMA buffer and is not readily

 * reclaimable. So those GFP bits should be masked off.

 */

#define OBJCGS_CLEAR_MASK	(__GFP_DMA | __GFP_RECLAIMABLE | \

				__GFP_ACCOUNT | __GFP_NOFAIL)



#ifdef CONFIG_SLAB_OBJ_EXT



#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
@@ -2081,14 +2089,6 @@ static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 


#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */



/*

 * The allocated objcg pointers array is not accounted directly.

 * Moreover, it should not come from DMA buffer and is not readily

 * reclaimable. So those GFP bits should be masked off.

 */

#define OBJCGS_CLEAR_MASK	(__GFP_DMA | __GFP_RECLAIMABLE | \

				__GFP_ACCOUNT | __GFP_NOFAIL)



static inline void init_slab_obj_exts(struct slab *slab)

{

	slab->obj_exts = 0;
@@ -2596,8 +2596,24 @@ static void *setup_object(struct kmem_cache *s, void *object) 


static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)

{

	struct slab_sheaf *sheaf = kzalloc(struct_size(sheaf, objects,

					s->sheaf_capacity), gfp);

	struct slab_sheaf *sheaf;

	size_t sheaf_size;



	if (gfp & __GFP_NO_OBJ_EXT)

		return NULL;



	gfp &= ~OBJCGS_CLEAR_MASK;



	/*

	 * Prevent recursion to the same cache, or a deep stack of kmallocs of

	 * varying sizes (sheaf capacity might differ for each kmalloc size

	 * bucket)

	 */

	if (s->flags & SLAB_KMALLOC)

		gfp |= __GFP_NO_OBJ_EXT;



	sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);

	sheaf = kzalloc(sheaf_size, gfp);



	if (unlikely(!sheaf))

		return NULL;