Commit 913ffd3a authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

slab: handle kmalloc sheaves bootstrap 



Enable sheaves for kmalloc caches. For other types than KMALLOC_NORMAL,
we can simply allow them in calculate_sizes() as they are created later
than KMALLOC_NORMAL caches and can allocate sheaves and barns from
those.

For KMALLOC_NORMAL caches we perform additional step after first
creating them without sheaves. Then bootstrap_cache_sheaves() simply
allocates and initializes barns and sheaves and finally sets
s->sheaf_capacity to make them actually used.

Afterwards the only caches left without sheaves (unless SLUB_TINY or
debugging is enabled) are kmem_cache and kmem_cache_node. These are only
used when creating or destroying other kmem_caches. Thus they are not
performance critical and we can simply leave it that way.

Reviewed-by: default avatarHarry Yoo <harry.yoo@oracle.com>
Reviewed-by: default avatarHao Li <hao.li@linux.dev>
Reviewed-by: default avatarLiam R. Howlett <Liam.Howlett@oracle.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent f1427a1d

mm/slub.c

+84 −4
Original line number Diff line number Diff line
@@ -2593,7 +2593,8 @@ static void *setup_object(struct kmem_cache *s, void *object) 
	return object;

}



static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)

static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,

					      unsigned int capacity)

{

	struct slab_sheaf *sheaf;

	size_t sheaf_size;
@@ -2611,7 +2612,7 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) 
	if (s->flags & SLAB_KMALLOC)

		gfp |= __GFP_NO_OBJ_EXT;



	sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);

	sheaf_size = struct_size(sheaf, objects, capacity);

	sheaf = kzalloc(sheaf_size, gfp);



	if (unlikely(!sheaf))
@@ -2624,6 +2625,12 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) 
	return sheaf;

}



static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,

						   gfp_t gfp)

{

	return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity);

}



static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)

{

	kfree(sheaf);
@@ -8157,8 +8164,11 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) 
	if (s->flags & SLAB_RECLAIM_ACCOUNT)

		s->allocflags |= __GFP_RECLAIMABLE;



	/* kmalloc caches need extra care to support sheaves */

	if (!is_kmalloc_cache(s))

	/*

	 * For KMALLOC_NORMAL caches we enable sheaves later by

	 * bootstrap_kmalloc_sheaves() to avoid recursion

	 */

	if (!is_kmalloc_normal(s))

		s->sheaf_capacity = calculate_sheaf_capacity(s, args);



	/*
@@ -8653,6 +8663,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) 
	return s;

}



/*

 * Finish the sheaves initialization done normally by init_percpu_sheaves() and

 * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it

 * since sheaves and barns are allocated by kmalloc.

 */

static void __init bootstrap_cache_sheaves(struct kmem_cache *s)

{

	struct kmem_cache_args empty_args = {};

	unsigned int capacity;

	bool failed = false;

	int node, cpu;



	capacity = calculate_sheaf_capacity(s, &empty_args);



	/* capacity can be 0 due to debugging or SLUB_TINY */

	if (!capacity)

		return;



	for_each_node_mask(node, slab_nodes) {

		struct node_barn *barn;



		barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);



		if (!barn) {

			failed = true;

			goto out;

		}



		barn_init(barn);

		get_node(s, node)->barn = barn;

	}



	for_each_possible_cpu(cpu) {

		struct slub_percpu_sheaves *pcs;



		pcs = per_cpu_ptr(s->cpu_sheaves, cpu);



		pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);



		if (!pcs->main) {

			failed = true;

			break;

		}

	}



out:

	/*

	 * It's still early in boot so treat this like same as a failure to

	 * create the kmalloc cache in the first place

	 */

	if (failed)

		panic("Out of memory when creating kmem_cache %s\n", s->name);



	s->sheaf_capacity = capacity;

}



static void __init bootstrap_kmalloc_sheaves(void)

{

	enum kmalloc_cache_type type;



	for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {

		for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {

			if (kmalloc_caches[type][idx])

				bootstrap_cache_sheaves(kmalloc_caches[type][idx]);

		}

	}

}



void __init kmem_cache_init(void)

{

	static __initdata struct kmem_cache boot_kmem_cache,
@@ -8696,6 +8774,8 @@ void __init kmem_cache_init(void) 
	setup_kmalloc_cache_index_table();

	create_kmalloc_caches();



	bootstrap_kmalloc_sheaves();



	/* Setup random freelists for each cache */

	init_freelist_randomization();