Commit 9a881ea3 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'slab-for-7.0-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab 

Pull slab fixes from Vlastimil Babka:

 - Fix for slab->stride truncation on 64k page systems due to short
   type. It was not due to races and lack of barriers in the end. (Harry
   Yoo)

 - Fix for severe performance regression due to unnecessary sheaf refill
   restrictions exposed by mempool allocation strategy. (Vlastimil
   Babka)

 - Stable fix for potential silent percpu sheaf flushing failures on
   PREEMPT_RT. (Vlastimil Babka)

* tag 'slab-for-7.0-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
  mm/slab: change stride type from unsigned short to unsigned int
  mm/slab: allow sheaf refill if blocking is not allowed
  slab: distinguish lock and trylock for sheaf_flush_main()
parents 345dfaaf 6432f15c

mm/slab.h

+5 −5
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ struct freelist_counters { 
					 * to save memory. In case ->stride field is not available,

					 * such optimizations are disabled.

					 */

					unsigned short stride;

					unsigned int stride;

#endif

				};

			};
@@ -559,20 +559,20 @@ static inline void put_slab_obj_exts(unsigned long obj_exts) 
}



#ifdef CONFIG_64BIT

static inline void slab_set_stride(struct slab *slab, unsigned short stride)

static inline void slab_set_stride(struct slab *slab, unsigned int stride)

{

	slab->stride = stride;

}

static inline unsigned short slab_get_stride(struct slab *slab)

static inline unsigned int slab_get_stride(struct slab *slab)

{

	return slab->stride;

}

#else

static inline void slab_set_stride(struct slab *slab, unsigned short stride)

static inline void slab_set_stride(struct slab *slab, unsigned int stride)

{

	VM_WARN_ON_ONCE(stride != sizeof(struct slabobj_ext));

}

static inline unsigned short slab_get_stride(struct slab *slab)

static inline unsigned int slab_get_stride(struct slab *slab)

{

	return sizeof(struct slabobj_ext);

}

mm/slub.c

+47 −22
Original line number Diff line number Diff line
@@ -2858,19 +2858,19 @@ static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); 
 * object pointers are moved to a on-stack array under the lock. To bound the

 * stack usage, limit each batch to PCS_BATCH_MAX.

 *

 * returns true if at least partially flushed

 * Must be called with s->cpu_sheaves->lock locked, returns with the lock

 * unlocked.

 *

 * Returns how many objects are remaining to be flushed

 */

static bool sheaf_flush_main(struct kmem_cache *s)

static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s)

{

	struct slub_percpu_sheaves *pcs;

	unsigned int batch, remaining;

	void *objects[PCS_BATCH_MAX];

	struct slab_sheaf *sheaf;

	bool ret = false;



next_batch:

	if (!local_trylock(&s->cpu_sheaves->lock))

		return ret;

	lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));



	pcs = this_cpu_ptr(s->cpu_sheaves);

	sheaf = pcs->main;
@@ -2888,10 +2888,37 @@ static bool sheaf_flush_main(struct kmem_cache *s) 


	stat_add(s, SHEAF_FLUSH, batch);



	return remaining;

}



static void sheaf_flush_main(struct kmem_cache *s)

{

	unsigned int remaining;



	do {

		local_lock(&s->cpu_sheaves->lock);



		remaining = __sheaf_flush_main_batch(s);



	} while (remaining);

}



/*

 * Returns true if the main sheaf was at least partially flushed.

 */

static bool sheaf_try_flush_main(struct kmem_cache *s)

{

	unsigned int remaining;

	bool ret = false;



	do {

		if (!local_trylock(&s->cpu_sheaves->lock))

			return ret;



		ret = true;

		remaining = __sheaf_flush_main_batch(s);



	if (remaining)

		goto next_batch;

	} while (remaining);



	return ret;

}
@@ -4540,7 +4567,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
	struct slab_sheaf *empty = NULL;

	struct slab_sheaf *full;

	struct node_barn *barn;

	bool can_alloc;

	bool allow_spin;



	lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
@@ -4561,8 +4588,9 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
		return NULL;

	}



	full = barn_replace_empty_sheaf(barn, pcs->main,

					gfpflags_allow_spinning(gfp));

	allow_spin = gfpflags_allow_spinning(gfp);



	full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);



	if (full) {

		stat(s, BARN_GET);
@@ -4572,9 +4600,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 


	stat(s, BARN_GET_FAIL);



	can_alloc = gfpflags_allow_blocking(gfp);



	if (can_alloc) {

	if (allow_spin) {

		if (pcs->spare) {

			empty = pcs->spare;

			pcs->spare = NULL;
@@ -4584,8 +4610,9 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
	}



	local_unlock(&s->cpu_sheaves->lock);

	pcs = NULL;



	if (!can_alloc)

	if (!allow_spin)

		return NULL;



	if (empty) {
@@ -4605,11 +4632,8 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
	if (!full)

		return NULL;



	/*

	 * we can reach here only when gfpflags_allow_blocking

	 * so this must not be an irq

	 */

	local_lock(&s->cpu_sheaves->lock);

	if (!local_trylock(&s->cpu_sheaves->lock))

		goto barn_put;

	pcs = this_cpu_ptr(s->cpu_sheaves);



	/*
@@ -4640,6 +4664,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
		return pcs;

	}



barn_put:

	barn_put_full_sheaf(barn, full);

	stat(s, BARN_PUT);
@@ -5704,7 +5729,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, 
	if (put_fail)

		 stat(s, BARN_PUT_FAIL);



	if (!sheaf_flush_main(s))

	if (!sheaf_try_flush_main(s))

		return NULL;



	if (!local_trylock(&s->cpu_sheaves->lock))