Commit ab2f752a authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

slab: remove defer_deactivate_slab() 



There are no more cpu slabs so we don't need their deferred
deactivation. The function is now only used from places where we
allocate a new slab but then can't spin on node list_lock to put it on
the partial list. Instead of the deferred action we can free it directly
via __free_slab(), we just need to tell it to use _nolock() freeing of
the underlying pages and take care of the accounting.

Since free_frozen_pages_nolock() variant does not yet exist for code
outside of the page allocator, create it as a trivial wrapper for
__free_frozen_pages(..., FPI_TRYLOCK).

Reviewed-by: default avatarHarry Yoo <harry.yoo@oracle.com>
Reviewed-by: default avatarHao Li <hao.li@linux.dev>
Reviewed-by: default avatarSuren Baghdasaryan <surenb@google.com>
Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent bdc9282f

mm/internal.h

+1 −0
Original line number Diff line number Diff line
@@ -846,6 +846,7 @@ static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int ord 
struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);

#define alloc_frozen_pages_nolock(...) \

	alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))

void free_frozen_pages_nolock(struct page *page, unsigned int order);



extern void zone_pcp_reset(struct zone *zone);

extern void zone_pcp_disable(struct zone *zone);

mm/page_alloc.c

+5 −0
Original line number Diff line number Diff line
@@ -2981,6 +2981,11 @@ void free_frozen_pages(struct page *page, unsigned int order) 
	__free_frozen_pages(page, order, FPI_NONE);

}



void free_frozen_pages_nolock(struct page *page, unsigned int order)

{

	__free_frozen_pages(page, order, FPI_TRYLOCK);

}



/*

 * Free a batch of folios

 */

mm/slab.h

+1 −7
Original line number Diff line number Diff line
@@ -71,13 +71,7 @@ struct slab { 
	struct kmem_cache *slab_cache;

	union {

		struct {

			union {

			struct list_head slab_list;

				struct { /* For deferred deactivate_slab() */

					struct llist_node llnode;

					void *flush_freelist;

				};

			};

			/* Double-word boundary */

			struct freelist_counters;

		};

mm/slub.c

+21 −37
Original line number Diff line number Diff line
@@ -3262,7 +3262,7 @@ static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) 
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);

}



static void __free_slab(struct kmem_cache *s, struct slab *slab)

static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)

{

	struct page *page = slab_page(slab);

	int order = compound_order(page);
@@ -3273,14 +3273,26 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) 
	__ClearPageSlab(page);

	mm_account_reclaimed_pages(pages);

	unaccount_slab(slab, order, s);

	if (allow_spin)

		free_frozen_pages(page, order);

	else

		free_frozen_pages_nolock(page, order);

}



static void free_new_slab_nolock(struct kmem_cache *s, struct slab *slab)

{

	/*

	 * Since it was just allocated, we can skip the actions in

	 * discard_slab() and free_slab().

	 */

	__free_slab(s, slab, false);

}



static void rcu_free_slab(struct rcu_head *h)

{

	struct slab *slab = container_of(h, struct slab, rcu_head);



	__free_slab(slab->slab_cache, slab);

	__free_slab(slab->slab_cache, slab, true);

}



static void free_slab(struct kmem_cache *s, struct slab *slab)
@@ -3296,7 +3308,7 @@ static void free_slab(struct kmem_cache *s, struct slab *slab) 
	if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))

		call_rcu(&slab->rcu_head, rcu_free_slab);

	else

		__free_slab(s, slab);

		__free_slab(s, slab, true);

}



static void discard_slab(struct kmem_cache *s, struct slab *slab)
@@ -3389,8 +3401,6 @@ static void *alloc_single_from_partial(struct kmem_cache *s, 
	return object;

}



static void defer_deactivate_slab(struct slab *slab, void *flush_freelist);



/*

 * Called only for kmem_cache_debug() caches to allocate from a freshly

 * allocated slab. Allocate a single object instead of whole freelist
@@ -3406,8 +3416,8 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, 
	void *object;



	if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) {

		/* Unlucky, discard newly allocated slab */

		defer_deactivate_slab(slab, NULL);

		/* Unlucky, discard newly allocated slab. */

		free_new_slab_nolock(s, slab);

		return NULL;

	}
@@ -4279,7 +4289,7 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab, 


		if (!spin_trylock_irqsave(&n->list_lock, flags)) {

			/* Unlucky, discard newly allocated slab */

			defer_deactivate_slab(slab, NULL);

			free_new_slab_nolock(s, slab);

			return 0;

		}

	}
@@ -6059,7 +6069,6 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 


struct defer_free {

	struct llist_head objects;

	struct llist_head slabs;

	struct irq_work work;

};
@@ -6067,23 +6076,21 @@ static void free_deferred_objects(struct irq_work *work); 


static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = {

	.objects = LLIST_HEAD_INIT(objects),

	.slabs = LLIST_HEAD_INIT(slabs),

	.work = IRQ_WORK_INIT(free_deferred_objects),

};



/*

 * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe

 * to take sleeping spin_locks from __slab_free() and deactivate_slab().

 * to take sleeping spin_locks from __slab_free().

 * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore().

 */

static void free_deferred_objects(struct irq_work *work)

{

	struct defer_free *df = container_of(work, struct defer_free, work);

	struct llist_head *objs = &df->objects;

	struct llist_head *slabs = &df->slabs;

	struct llist_node *llnode, *pos, *t;



	if (llist_empty(objs) && llist_empty(slabs))

	if (llist_empty(objs))

		return;



	llnode = llist_del_all(objs);
@@ -6107,16 +6114,6 @@ static void free_deferred_objects(struct irq_work *work) 


		__slab_free(s, slab, x, x, 1, _THIS_IP_);

	}



	llnode = llist_del_all(slabs);

	llist_for_each_safe(pos, t, llnode) {

		struct slab *slab = container_of(pos, struct slab, llnode);



		if (slab->frozen)

			deactivate_slab(slab->slab_cache, slab, slab->flush_freelist);

		else

			free_slab(slab->slab_cache, slab);

	}

}



static void defer_free(struct kmem_cache *s, void *head)
@@ -6132,19 +6129,6 @@ static void defer_free(struct kmem_cache *s, void *head) 
		irq_work_queue(&df->work);

}



static void defer_deactivate_slab(struct slab *slab, void *flush_freelist)

{

	struct defer_free *df;



	slab->flush_freelist = flush_freelist;



	guard(preempt)();



	df = this_cpu_ptr(&defer_free_objects);

	if (llist_add(&slab->llnode, &df->slabs))

		irq_work_queue(&df->work);

}



void defer_free_barrier(void)

{

	int cpu;