Commit 6bb71f0f authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'slab-for-6.18-rc1-hotfix' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab 

Pull slab fix from Vlastimil Babka:
 "A NULL pointer deref hotfix"

* tag 'slab-for-6.18-rc1-hotfix' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
  slab: fix barn NULL pointer dereference on memoryless nodes
parents fbde105f fd6db588

mm/slub.c

+51 −14
Original line number Diff line number Diff line
@@ -504,10 +504,18 @@ static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 
	return s->node[node];

}



/* Get the barn of the current cpu's memory node */

/*

 * Get the barn of the current cpu's closest memory node. It may not exist on

 * systems with memoryless nodes but without CONFIG_HAVE_MEMORYLESS_NODES

 */

static inline struct node_barn *get_barn(struct kmem_cache *s)

{

	return get_node(s, numa_mem_id())->barn;

	struct kmem_cache_node *n = get_node(s, numa_mem_id());



	if (!n)

		return NULL;



	return n->barn;

}



/*
@@ -4982,6 +4990,10 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
	}



	barn = get_barn(s);

	if (!barn) {

		local_unlock(&s->cpu_sheaves->lock);

		return NULL;

	}



	full = barn_replace_empty_sheaf(barn, pcs->main);
@@ -5153,13 +5165,20 @@ unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 
	if (unlikely(pcs->main->size == 0)) {



		struct slab_sheaf *full;

		struct node_barn *barn;



		if (pcs->spare && pcs->spare->size > 0) {

			swap(pcs->main, pcs->spare);

			goto do_alloc;

		}



		full = barn_replace_empty_sheaf(get_barn(s), pcs->main);

		barn = get_barn(s);

		if (!barn) {

			local_unlock(&s->cpu_sheaves->lock);

			return allocated;

		}



		full = barn_replace_empty_sheaf(barn, pcs->main);



		if (full) {

			stat(s, BARN_GET);
@@ -5314,6 +5333,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) 
{

	struct slub_percpu_sheaves *pcs;

	struct slab_sheaf *sheaf = NULL;

	struct node_barn *barn;



	if (unlikely(size > s->sheaf_capacity)) {
@@ -5355,8 +5375,11 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) 
		pcs->spare = NULL;

		stat(s, SHEAF_PREFILL_FAST);

	} else {

		barn = get_barn(s);



		stat(s, SHEAF_PREFILL_SLOW);

		sheaf = barn_get_full_or_empty_sheaf(get_barn(s));

		if (barn)

			sheaf = barn_get_full_or_empty_sheaf(barn);

		if (sheaf && sheaf->size)

			stat(s, BARN_GET);

		else
@@ -5426,7 +5449,7 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, 
	 * If the barn has too many full sheaves or we fail to refill the sheaf,

	 * simply flush and free it.

	 */

	if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES ||

	if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES ||

	    refill_sheaf(s, sheaf, gfp)) {

		sheaf_flush_unused(s, sheaf);

		free_empty_sheaf(s, sheaf);
@@ -5943,10 +5966,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, 
 * put the full sheaf there.

 */

static void __pcs_install_empty_sheaf(struct kmem_cache *s,

		struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty)

		struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty,

		struct node_barn *barn)

{

	struct node_barn *barn;



	lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));



	/* This is what we expect to find if nobody interrupted us. */
@@ -5956,8 +5978,6 @@ static void __pcs_install_empty_sheaf(struct kmem_cache *s, 
		return;

	}



	barn = get_barn(s);



	/*

	 * Unlikely because if the main sheaf had space, we would have just

	 * freed to it. Get rid of our empty sheaf.
@@ -6002,6 +6022,11 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) 
	lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));



	barn = get_barn(s);

	if (!barn) {

		local_unlock(&s->cpu_sheaves->lock);

		return NULL;

	}



	put_fail = false;



	if (!pcs->spare) {
@@ -6084,7 +6109,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) 
	}



	pcs = this_cpu_ptr(s->cpu_sheaves);

	__pcs_install_empty_sheaf(s, pcs, empty);

	__pcs_install_empty_sheaf(s, pcs, empty, barn);



	return pcs;

}
@@ -6121,8 +6146,9 @@ bool free_to_pcs(struct kmem_cache *s, void *object) 


static void rcu_free_sheaf(struct rcu_head *head)

{

	struct kmem_cache_node *n;

	struct slab_sheaf *sheaf;

	struct node_barn *barn;

	struct node_barn *barn = NULL;

	struct kmem_cache *s;



	sheaf = container_of(head, struct slab_sheaf, rcu_head);
@@ -6139,7 +6165,11 @@ static void rcu_free_sheaf(struct rcu_head *head) 
	 */

	__rcu_free_sheaf_prepare(s, sheaf);



	barn = get_node(s, sheaf->node)->barn;

	n = get_node(s, sheaf->node);

	if (!n)

		goto flush;



	barn = n->barn;



	/* due to slab_free_hook() */

	if (unlikely(sheaf->size == 0))
@@ -6157,11 +6187,12 @@ static void rcu_free_sheaf(struct rcu_head *head) 
		return;

	}



flush:

	stat(s, BARN_PUT_FAIL);

	sheaf_flush_unused(s, sheaf);



empty:

	if (data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) {

	if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) {

		barn_put_empty_sheaf(barn, sheaf);

		return;

	}
@@ -6191,6 +6222,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) 
		}



		barn = get_barn(s);

		if (!barn) {

			local_unlock(&s->cpu_sheaves->lock);

			goto fail;

		}



		empty = barn_get_empty_sheaf(barn);
@@ -6304,6 +6339,8 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 
		goto do_free;



	barn = get_barn(s);

	if (!barn)

		goto no_empty;



	if (!pcs->spare) {

		empty = barn_get_empty_sheaf(barn);