Commit 1de74115 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'slab-for-6.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab 

Pull slab fix from Vlastimil Babka:

 - A stable fix for performance regression in tests that perform
   kmem_cache_destroy() a lot, due to unnecessarily wide scope of
   kvfree_rcu_barrier() (Harry Yoo)

* tag 'slab-for-6.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
  mm/slab: introduce kvfree_rcu_barrier_on_cache() for cache destruction
parents 0723a166 0f35040d

include/linux/slab.h

+7 −0
Original line number Diff line number Diff line
@@ -1150,10 +1150,17 @@ static inline void kvfree_rcu_barrier(void) 
	rcu_barrier();

}



static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)

{

	rcu_barrier();

}



static inline void kfree_rcu_scheduler_running(void) { }

#else

void kvfree_rcu_barrier(void);



void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);



void kfree_rcu_scheduler_running(void);

#endif

mm/slab.h

+1 −0
Original line number Diff line number Diff line
@@ -422,6 +422,7 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) 


bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj);

void flush_all_rcu_sheaves(void);

void flush_rcu_sheaves_on_cache(struct kmem_cache *s);



#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \

			 SLAB_CACHE_DMA32 | SLAB_PANIC | \

mm/slab_common.c

+37 −15
Original line number Diff line number Diff line
@@ -492,7 +492,7 @@ void kmem_cache_destroy(struct kmem_cache *s) 
		return;



	/* in-flight kfree_rcu()'s may include objects from our cache */

	kvfree_rcu_barrier();

	kvfree_rcu_barrier_on_cache(s);



	if (IS_ENABLED(CONFIG_SLUB_RCU_DEBUG) &&

	    (s->flags & SLAB_TYPESAFE_BY_RCU)) {
@@ -2038,25 +2038,13 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) 
}

EXPORT_SYMBOL_GPL(kvfree_call_rcu);



/**

 * kvfree_rcu_barrier - Wait until all in-flight kvfree_rcu() complete.

 *

 * Note that a single argument of kvfree_rcu() call has a slow path that

 * triggers synchronize_rcu() following by freeing a pointer. It is done

 * before the return from the function. Therefore for any single-argument

 * call that will result in a kfree() to a cache that is to be destroyed

 * during module exit, it is developer's responsibility to ensure that all

 * such calls have returned before the call to kmem_cache_destroy().

 */

void kvfree_rcu_barrier(void)

static inline void __kvfree_rcu_barrier(void)

{

	struct kfree_rcu_cpu_work *krwp;

	struct kfree_rcu_cpu *krcp;

	bool queued;

	int i, cpu;



	flush_all_rcu_sheaves();



	/*

	 * Firstly we detach objects and queue them over an RCU-batch

	 * for all CPUs. Finally queued works are flushed for each CPU.
@@ -2118,8 +2106,43 @@ void kvfree_rcu_barrier(void) 
		}

	}

}



/**

 * kvfree_rcu_barrier - Wait until all in-flight kvfree_rcu() complete.

 *

 * Note that a single argument of kvfree_rcu() call has a slow path that

 * triggers synchronize_rcu() following by freeing a pointer. It is done

 * before the return from the function. Therefore for any single-argument

 * call that will result in a kfree() to a cache that is to be destroyed

 * during module exit, it is developer's responsibility to ensure that all

 * such calls have returned before the call to kmem_cache_destroy().

 */

void kvfree_rcu_barrier(void)

{

	flush_all_rcu_sheaves();

	__kvfree_rcu_barrier();

}

EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);



/**

 * kvfree_rcu_barrier_on_cache - Wait for in-flight kvfree_rcu() calls on a

 *                               specific slab cache.

 * @s: slab cache to wait for

 *

 * See the description of kvfree_rcu_barrier() for details.

 */

void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)

{

	if (s->cpu_sheaves)

		flush_rcu_sheaves_on_cache(s);

	/*

	 * TODO: Introduce a version of __kvfree_rcu_barrier() that works

	 * on a specific slab cache.

	 */

	__kvfree_rcu_barrier();

}

EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache);



static unsigned long

kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)

{
@@ -2215,4 +2238,3 @@ void __init kvfree_rcu_init(void) 
}



#endif /* CONFIG_KVFREE_RCU_BATCHED */

mm/slub.c

+30 −25
Original line number Diff line number Diff line
@@ -4122,19 +4122,11 @@ static void flush_rcu_sheaf(struct work_struct *w) 




/* needed for kvfree_rcu_barrier() */

void flush_all_rcu_sheaves(void)

void flush_rcu_sheaves_on_cache(struct kmem_cache *s)

{

	struct slub_flush_work *sfw;

	struct kmem_cache *s;

	unsigned int cpu;



	cpus_read_lock();

	mutex_lock(&slab_mutex);



	list_for_each_entry(s, &slab_caches, list) {

		if (!s->cpu_sheaves)

			continue;



	mutex_lock(&flush_lock);



	for_each_online_cpu(cpu) {
@@ -4160,6 +4152,19 @@ void flush_all_rcu_sheaves(void) 
	mutex_unlock(&flush_lock);

}



void flush_all_rcu_sheaves(void)

{

	struct kmem_cache *s;



	cpus_read_lock();

	mutex_lock(&slab_mutex);



	list_for_each_entry(s, &slab_caches, list) {

		if (!s->cpu_sheaves)

			continue;

		flush_rcu_sheaves_on_cache(s);

	}



	mutex_unlock(&slab_mutex);

	cpus_read_unlock();