Commit ec66e0d5 authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

slab: add sheaf support for batching kfree_rcu() operations 



Extend the sheaf infrastructure for more efficient kfree_rcu() handling.
For caches with sheaves, on each cpu maintain a rcu_free sheaf in
addition to main and spare sheaves.

kfree_rcu() operations will try to put objects on this sheaf. Once full,
the sheaf is detached and submitted to call_rcu() with a handler that
will try to put it in the barn, or flush to slab pages using bulk free,
when the barn is full. Then a new empty sheaf must be obtained to put
more objects there.

It's possible that no free sheaves are available to use for a new
rcu_free sheaf, and the allocation in kfree_rcu() context can only use
GFP_NOWAIT and thus may fail. In that case, fall back to the existing
kfree_rcu() implementation.

Expected advantages:
- batching the kfree_rcu() operations, that could eventually replace the
  existing batching
- sheaves can be reused for allocations via barn instead of being
  flushed to slabs, which is more efficient
  - this includes cases where only some cpus are allowed to process rcu
    callbacks (CONFIG_RCU_NOCB_CPU)

Possible disadvantage:
- objects might be waiting for more than their grace period (it is
  determined by the last object freed into the sheaf), increasing memory
  usage - but the existing batching does that too.

Only implement this for CONFIG_KVFREE_RCU_BATCHED as the tiny
implementation favors smaller memory footprint over performance.

Also for now skip the usage of rcu sheaf for CONFIG_PREEMPT_RT as the
contexts where kfree_rcu() is called might not be compatible with taking
a barn spinlock or a GFP_NOWAIT allocation of a new sheaf taking a
spinlock - the current kfree_rcu() implementation avoids doing that.

Teach kvfree_rcu_barrier() to flush all rcu_free sheaves from all caches
that have them. This is not a cheap operation, but the barrier usage is
rare - currently kmem_cache_destroy() or on module unload.

Add CONFIG_SLUB_STATS counters free_rcu_sheaf and free_rcu_sheaf_fail to
count how many kfree_rcu() used the rcu_free sheaf successfully and how
many had to fall back to the existing implementation.

Reviewed-by: default avatarHarry Yoo <harry.yoo@oracle.com>
Reviewed-by: default avatarSuren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 2d517aa0

mm/slab.h

+3 −0
Original line number Diff line number Diff line
@@ -435,6 +435,9 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) 
	return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT));

}



bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj);

void flush_all_rcu_sheaves(void);



#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \

			 SLAB_CACHE_DMA32 | SLAB_PANIC | \

			 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS | \

mm/slab_common.c

+26 −0
Original line number Diff line number Diff line
@@ -1608,6 +1608,27 @@ static void kfree_rcu_work(struct work_struct *work) 
		kvfree_rcu_list(head);

}



static bool kfree_rcu_sheaf(void *obj)

{

	struct kmem_cache *s;

	struct folio *folio;

	struct slab *slab;



	if (is_vmalloc_addr(obj))

		return false;



	folio = virt_to_folio(obj);

	if (unlikely(!folio_test_slab(folio)))

		return false;



	slab = folio_slab(folio);

	s = slab->slab_cache;

	if (s->cpu_sheaves)

		return __kfree_rcu_sheaf(s, obj);



	return false;

}



static bool

need_offload_krc(struct kfree_rcu_cpu *krcp)

{
@@ -1952,6 +1973,9 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr) 
	if (!head)

		might_sleep();



	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && kfree_rcu_sheaf(ptr))

		return;



	// Queue the object but don't yet schedule the batch.

	if (debug_rcu_head_queue(ptr)) {

		// Probable double kfree_rcu(), just leak.
@@ -2026,6 +2050,8 @@ void kvfree_rcu_barrier(void) 
	bool queued;

	int i, cpu;



	flush_all_rcu_sheaves();



	/*

	 * Firstly we detach objects and queue them over an RCU-batch

	 * for all CPUs. Finally queued works are flushed for each CPU.

mm/slub.c

+266 −2
Original line number Diff line number Diff line
@@ -367,6 +367,8 @@ enum stat_item { 
	ALLOC_FASTPATH,		/* Allocation from cpu slab */

	ALLOC_SLOWPATH,		/* Allocation by getting a new cpu slab */

	FREE_PCS,		/* Free to percpu sheaf */

	FREE_RCU_SHEAF,		/* Free to rcu_free sheaf */

	FREE_RCU_SHEAF_FAIL,	/* Failed to free to a rcu_free sheaf */

	FREE_FASTPATH,		/* Free to cpu slab */

	FREE_SLOWPATH,		/* Freeing not to cpu slab */

	FREE_FROZEN,		/* Freeing to frozen slab */
@@ -461,6 +463,7 @@ struct slab_sheaf { 
		struct rcu_head rcu_head;

		struct list_head barn_list;

	};

	struct kmem_cache *cache;

	unsigned int size;

	void *objects[];

};
@@ -469,6 +472,7 @@ struct slub_percpu_sheaves { 
	local_trylock_t lock;

	struct slab_sheaf *main; /* never NULL when unlocked */

	struct slab_sheaf *spare; /* empty or full, may be NULL */

	struct slab_sheaf *rcu_free; /* for batching kfree_rcu() */

};



/*
@@ -2531,6 +2535,8 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) 
	if (unlikely(!sheaf))

		return NULL;



	sheaf->cache = s;



	stat(s, SHEAF_ALLOC);



	return sheaf;
@@ -2655,6 +2661,43 @@ static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf) 
	sheaf->size = 0;

}



static void __rcu_free_sheaf_prepare(struct kmem_cache *s,

				     struct slab_sheaf *sheaf)

{

	bool init = slab_want_init_on_free(s);

	void **p = &sheaf->objects[0];

	unsigned int i = 0;



	while (i < sheaf->size) {

		struct slab *slab = virt_to_slab(p[i]);



		memcg_slab_free_hook(s, slab, p + i, 1);

		alloc_tagging_slab_free_hook(s, slab, p + i, 1);



		if (unlikely(!slab_free_hook(s, p[i], init, true))) {

			p[i] = p[--sheaf->size];

			continue;

		}



		i++;

	}

}



static void rcu_free_sheaf_nobarn(struct rcu_head *head)

{

	struct slab_sheaf *sheaf;

	struct kmem_cache *s;



	sheaf = container_of(head, struct slab_sheaf, rcu_head);

	s = sheaf->cache;



	__rcu_free_sheaf_prepare(s, sheaf);



	sheaf_flush_unused(s, sheaf);



	free_empty_sheaf(s, sheaf);

}



/*

 * Caller needs to make sure migration is disabled in order to fully flush

 * single cpu's sheaves
@@ -2667,7 +2710,7 @@ static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf) 
static void pcs_flush_all(struct kmem_cache *s)

{

	struct slub_percpu_sheaves *pcs;

	struct slab_sheaf *spare;

	struct slab_sheaf *spare, *rcu_free;



	local_lock(&s->cpu_sheaves->lock);

	pcs = this_cpu_ptr(s->cpu_sheaves);
@@ -2675,6 +2718,9 @@ static void pcs_flush_all(struct kmem_cache *s) 
	spare = pcs->spare;

	pcs->spare = NULL;



	rcu_free = pcs->rcu_free;

	pcs->rcu_free = NULL;



	local_unlock(&s->cpu_sheaves->lock);



	if (spare) {
@@ -2682,6 +2728,9 @@ static void pcs_flush_all(struct kmem_cache *s) 
		free_empty_sheaf(s, spare);

	}



	if (rcu_free)

		call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn);



	sheaf_flush_main(s);

}
@@ -2698,6 +2747,11 @@ static void __pcs_flush_all_cpu(struct kmem_cache *s, unsigned int cpu) 
		free_empty_sheaf(s, pcs->spare);

		pcs->spare = NULL;

	}



	if (pcs->rcu_free) {

		call_rcu(&pcs->rcu_free->rcu_head, rcu_free_sheaf_nobarn);

		pcs->rcu_free = NULL;

	}

}



static void pcs_destroy(struct kmem_cache *s)
@@ -2723,6 +2777,7 @@ static void pcs_destroy(struct kmem_cache *s) 
		 */



		WARN_ON(pcs->spare);

		WARN_ON(pcs->rcu_free);



		if (!WARN_ON(pcs->main->size)) {

			free_empty_sheaf(s, pcs->main);
@@ -3780,7 +3835,7 @@ static bool has_pcs_used(int cpu, struct kmem_cache *s) 


	pcs = per_cpu_ptr(s->cpu_sheaves, cpu);



	return (pcs->spare || pcs->main->size);

	return (pcs->spare || pcs->rcu_free || pcs->main->size);

}



/*
@@ -3840,6 +3895,74 @@ static void flush_all(struct kmem_cache *s) 
	cpus_read_unlock();

}



static void flush_rcu_sheaf(struct work_struct *w)

{

	struct slub_percpu_sheaves *pcs;

	struct slab_sheaf *rcu_free;

	struct slub_flush_work *sfw;

	struct kmem_cache *s;



	sfw = container_of(w, struct slub_flush_work, work);

	s = sfw->s;



	local_lock(&s->cpu_sheaves->lock);

	pcs = this_cpu_ptr(s->cpu_sheaves);



	rcu_free = pcs->rcu_free;

	pcs->rcu_free = NULL;



	local_unlock(&s->cpu_sheaves->lock);



	if (rcu_free)

		call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn);

}





/* needed for kvfree_rcu_barrier() */

void flush_all_rcu_sheaves(void)

{

	struct slub_flush_work *sfw;

	struct kmem_cache *s;

	unsigned int cpu;



	cpus_read_lock();

	mutex_lock(&slab_mutex);



	list_for_each_entry(s, &slab_caches, list) {

		if (!s->cpu_sheaves)

			continue;



		mutex_lock(&flush_lock);



		for_each_online_cpu(cpu) {

			sfw = &per_cpu(slub_flush, cpu);



			/*

			 * we don't check if rcu_free sheaf exists - racing

			 * __kfree_rcu_sheaf() might have just removed it.

			 * by executing flush_rcu_sheaf() on the cpu we make

			 * sure the __kfree_rcu_sheaf() finished its call_rcu()

			 */



			INIT_WORK(&sfw->work, flush_rcu_sheaf);

			sfw->s = s;

			queue_work_on(cpu, flushwq, &sfw->work);

		}



		for_each_online_cpu(cpu) {

			sfw = &per_cpu(slub_flush, cpu);

			flush_work(&sfw->work);

		}



		mutex_unlock(&flush_lock);

	}



	mutex_unlock(&slab_mutex);

	cpus_read_unlock();



	rcu_barrier();

}



/*

 * Use the cpu notifier to insure that the cpu slabs are flushed when

 * necessary.
@@ -5413,6 +5536,138 @@ bool free_to_pcs(struct kmem_cache *s, void *object) 
	return true;

}



static void rcu_free_sheaf(struct rcu_head *head)

{

	struct slab_sheaf *sheaf;

	struct node_barn *barn;

	struct kmem_cache *s;



	sheaf = container_of(head, struct slab_sheaf, rcu_head);



	s = sheaf->cache;



	/*

	 * This may remove some objects due to slab_free_hook() returning false,

	 * so that the sheaf might no longer be completely full. But it's easier

	 * to handle it as full (unless it became completely empty), as the code

	 * handles it fine. The only downside is that sheaf will serve fewer

	 * allocations when reused. It only happens due to debugging, which is a

	 * performance hit anyway.

	 */

	__rcu_free_sheaf_prepare(s, sheaf);



	barn = get_node(s, numa_mem_id())->barn;



	/* due to slab_free_hook() */

	if (unlikely(sheaf->size == 0))

		goto empty;



	/*

	 * Checking nr_full/nr_empty outside lock avoids contention in case the

	 * barn is at the respective limit. Due to the race we might go over the

	 * limit but that should be rare and harmless.

	 */



	if (data_race(barn->nr_full) < MAX_FULL_SHEAVES) {

		stat(s, BARN_PUT);

		barn_put_full_sheaf(barn, sheaf);

		return;

	}



	stat(s, BARN_PUT_FAIL);

	sheaf_flush_unused(s, sheaf);



empty:

	if (data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) {

		barn_put_empty_sheaf(barn, sheaf);

		return;

	}



	free_empty_sheaf(s, sheaf);

}



bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj)

{

	struct slub_percpu_sheaves *pcs;

	struct slab_sheaf *rcu_sheaf;



	if (!local_trylock(&s->cpu_sheaves->lock))

		goto fail;



	pcs = this_cpu_ptr(s->cpu_sheaves);



	if (unlikely(!pcs->rcu_free)) {



		struct slab_sheaf *empty;

		struct node_barn *barn;



		if (pcs->spare && pcs->spare->size == 0) {

			pcs->rcu_free = pcs->spare;

			pcs->spare = NULL;

			goto do_free;

		}



		barn = get_barn(s);



		empty = barn_get_empty_sheaf(barn);



		if (empty) {

			pcs->rcu_free = empty;

			goto do_free;

		}



		local_unlock(&s->cpu_sheaves->lock);



		empty = alloc_empty_sheaf(s, GFP_NOWAIT);



		if (!empty)

			goto fail;



		if (!local_trylock(&s->cpu_sheaves->lock)) {

			barn_put_empty_sheaf(barn, empty);

			goto fail;

		}



		pcs = this_cpu_ptr(s->cpu_sheaves);



		if (unlikely(pcs->rcu_free))

			barn_put_empty_sheaf(barn, empty);

		else

			pcs->rcu_free = empty;

	}



do_free:



	rcu_sheaf = pcs->rcu_free;



	/*

	 * Since we flush immediately when size reaches capacity, we never reach

	 * this with size already at capacity, so no OOB write is possible.

	 */

	rcu_sheaf->objects[rcu_sheaf->size++] = obj;



	if (likely(rcu_sheaf->size < s->sheaf_capacity))

		rcu_sheaf = NULL;

	else

		pcs->rcu_free = NULL;



	/*

	 * we flush before local_unlock to make sure a racing

	 * flush_all_rcu_sheaves() doesn't miss this sheaf

	 */

	if (rcu_sheaf)

		call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);



	local_unlock(&s->cpu_sheaves->lock);



	stat(s, FREE_RCU_SHEAF);

	return true;



fail:

	stat(s, FREE_RCU_SHEAF_FAIL);

	return false;

}



/*

 * Bulk free objects to the percpu sheaves.

 * Unlike free_to_pcs() this includes the calls to all necessary hooks
@@ -6909,6 +7164,11 @@ int __kmem_cache_shutdown(struct kmem_cache *s) 
	struct kmem_cache_node *n;



	flush_all_cpus_locked(s);



	/* we might have rcu sheaves in flight */

	if (s->cpu_sheaves)

		rcu_barrier();



	/* Attempt to free all objects */

	for_each_kmem_cache_node(s, node, n) {

		if (n->barn)
@@ -8284,6 +8544,8 @@ STAT_ATTR(ALLOC_PCS, alloc_cpu_sheaf); 
STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);

STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);

STAT_ATTR(FREE_PCS, free_cpu_sheaf);

STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf);

STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail);

STAT_ATTR(FREE_FASTPATH, free_fastpath);

STAT_ATTR(FREE_SLOWPATH, free_slowpath);

STAT_ATTR(FREE_FROZEN, free_frozen);
@@ -8382,6 +8644,8 @@ static struct attribute *slab_attrs[] = { 
	&alloc_fastpath_attr.attr,

	&alloc_slowpath_attr.attr,

	&free_cpu_sheaf_attr.attr,

	&free_rcu_sheaf_attr.attr,

	&free_rcu_sheaf_fail_attr.attr,

	&free_fastpath_attr.attr,

	&free_slowpath_attr.attr,

	&free_frozen_attr.attr,