Commit e5d7764e authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

Merge branch 'slab/for-6.19/memdesc_prep' into slab/for-next 

Merge series "Prepare slab for memdescs" by Matthew Wilcox.

From the cover letter [1]:

When we separate struct folio, struct page and struct slab from each
other, converting to folios then to slabs will be nonsense.  It made
sense under the 'folio is just a head page' interpretation, but with
full separation, page_folio() will return NULL for a page which belongs
to a slab.

This patch series removes almost all mentions of folio from slab.
There are a few folio_test_slab() invocations left around the tree that
I haven't decided how to handle yet.  We're not yet quite at the point
of separately allocating struct slab, but that's what I'll be working
on next.

Link: https://lore.kernel.org/all/20251113000932.1589073-1-willy@infradead.org/ [1]
parents 3065c20d b5559055

include/linux/page-flags.h

+2 −14
Original line number Diff line number Diff line
@@ -1048,19 +1048,7 @@ PAGE_TYPE_OPS(Table, table, pgtable) 
 */

PAGE_TYPE_OPS(Guard, guard, guard)



FOLIO_TYPE_OPS(slab, slab)



/**

 * PageSlab - Determine if the page belongs to the slab allocator

 * @page: The page to test.

 *

 * Context: Any context.

 * Return: True for slab pages, false for any other kind of page.

 */

static inline bool PageSlab(const struct page *page)

{

	return folio_test_slab(page_folio(page));

}

PAGE_TYPE_OPS(Slab, slab, slab)



#ifdef CONFIG_HUGETLB_PAGE

FOLIO_TYPE_OPS(hugetlb, hugetlb)
@@ -1076,7 +1064,7 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc) 
 * Serialized with zone lock.

 */

PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)

FOLIO_TYPE_OPS(large_kmalloc, large_kmalloc)

PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)



/**

 * PageHuge - Determine if the page belongs to hugetlbfs

mm/kasan/common.c

+4 −8
Original line number Diff line number Diff line
@@ -520,24 +520,20 @@ void __kasan_mempool_unpoison_pages(struct page *page, unsigned int order, 


bool __kasan_mempool_poison_object(void *ptr, unsigned long ip)

{

	struct folio *folio = virt_to_folio(ptr);

	struct page *page = virt_to_page(ptr);

	struct slab *slab;



	/*

	 * This function can be called for large kmalloc allocation that get

	 * their memory from page_alloc. Thus, the folio might not be a slab.

	 */

	if (unlikely(!folio_test_slab(folio))) {

	if (unlikely(PageLargeKmalloc(page))) {

		if (check_page_allocation(ptr, ip))

			return false;

		kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false);

		kasan_poison(ptr, page_size(page), KASAN_PAGE_FREE, false);

		return true;

	}



	if (is_kfence_address(ptr))

		return true;



	slab = folio_slab(folio);

	slab = page_slab(page);



	if (check_slab_allocation(slab->slab_cache, ptr, ip))

		return false;

mm/kfence/core.c

+8 −6
Original line number Diff line number Diff line
@@ -612,14 +612,15 @@ static unsigned long kfence_init_pool(void) 
	 * enters __slab_free() slow-path.

	 */

	for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {

		struct slab *slab;

		struct page *page;



		if (!i || (i % 2))

			continue;



		slab = page_slab(pfn_to_page(start_pfn + i));

		__folio_set_slab(slab_folio(slab));

		page = pfn_to_page(start_pfn + i);

		__SetPageSlab(page);

#ifdef CONFIG_MEMCG

		struct slab *slab = page_slab(page);

		slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts |

				 MEMCG_DATA_OBJEXTS;

#endif
@@ -665,16 +666,17 @@ static unsigned long kfence_init_pool(void) 


reset_slab:

	for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {

		struct slab *slab;

		struct page *page;



		if (!i || (i % 2))

			continue;



		slab = page_slab(pfn_to_page(start_pfn + i));

		page = pfn_to_page(start_pfn + i);

#ifdef CONFIG_MEMCG

		struct slab *slab = page_slab(page);

		slab->obj_exts = 0;

#endif

		__folio_clear_slab(slab_folio(slab));

		__ClearPageSlab(page);

	}



	return addr;

mm/memcontrol.c

+16 −24
Original line number Diff line number Diff line
@@ -2557,19 +2557,16 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, 
}



static __always_inline

struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p)

struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)

{

	/*

	 * Slab objects are accounted individually, not per-page.

	 * Memcg membership data for each individual object is saved in

	 * slab->obj_exts.

	 */

	if (folio_test_slab(folio)) {

	struct slabobj_ext *obj_exts;

		struct slab *slab;

	unsigned int off;



		slab = folio_slab(folio);

	obj_exts = slab_obj_exts(slab);

	if (!obj_exts)

		return NULL;
@@ -2581,16 +2578,6 @@ struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) 
	return NULL;

}



	/*

	 * folio_memcg_check() is used here, because in theory we can encounter

	 * a folio where the slab flag has been cleared already, but

	 * slab->obj_exts has not been freed yet

	 * folio_memcg_check() will guarantee that a proper memory

	 * cgroup pointer or NULL will be returned.

	 */

	return folio_memcg_check(folio);

}



/*

 * Returns a pointer to the memory cgroup to which the kernel object is charged.

 * It is not suitable for objects allocated using vmalloc().
@@ -2602,10 +2589,15 @@ struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) 
 */

struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)

{

	struct slab *slab;



	if (mem_cgroup_disabled())

		return NULL;



	return mem_cgroup_from_obj_folio(virt_to_folio(p), p);

	slab = virt_to_slab(p);

	if (slab)

		return mem_cgroup_from_obj_slab(slab, p);

	return folio_memcg_check(virt_to_folio(p));

}



static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)

mm/slab.h

+27 −31
Original line number Diff line number Diff line
@@ -117,19 +117,6 @@ static_assert(sizeof(struct slab) <= sizeof(struct page)); 
static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)));

#endif



/**

 * folio_slab - Converts from folio to slab.

 * @folio: The folio.

 *

 * Currently struct slab is a different representation of a folio where

 * folio_test_slab() is true.

 *

 * Return: The slab which contains this folio.

 */

#define folio_slab(folio)	(_Generic((folio),			\

	const struct folio *:	(const struct slab *)(folio),		\

	struct folio *:		(struct slab *)(folio)))



/**

 * slab_folio - The folio allocated for a slab

 * @s: The slab.
@@ -146,20 +133,24 @@ static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t) 
	struct slab *:		(struct folio *)s))



/**

 * page_slab - Converts from first struct page to slab.

 * @p: The first (either head of compound or single) page of slab.

 * page_slab - Converts from struct page to its slab.

 * @page: A page which may or may not belong to a slab.

 *

 * A temporary wrapper to convert struct page to struct slab in situations where

 * we know the page is the compound head, or single order-0 page.

 *

 * Long-term ideally everything would work with struct slab directly or go

 * through folio to struct slab.

 *

 * Return: The slab which contains this page

 * Return: The slab which contains this page or NULL if the page does

 * not belong to a slab.  This includes pages returned from large kmalloc.

 */

#define page_slab(p)		(_Generic((p),				\

	const struct page *:	(const struct slab *)(p),		\

	struct page *:		(struct slab *)(p)))

static inline struct slab *page_slab(const struct page *page)

{

	unsigned long head;



	head = READ_ONCE(page->compound_head);

	if (head & 1)

		page = (struct page *)(head - 1);

	if (data_race(page->page_type >> 24) != PGTY_slab)

		page = NULL;



	return (struct slab *)page;

}



/**

 * slab_page - The first struct page allocated for a slab
@@ -188,12 +179,7 @@ static inline pg_data_t *slab_pgdat(const struct slab *slab) 


static inline struct slab *virt_to_slab(const void *addr)

{

	struct folio *folio = virt_to_folio(addr);



	if (!folio_test_slab(folio))

		return NULL;



	return folio_slab(folio);

	return page_slab(virt_to_page(addr));

}



static inline int slab_order(const struct slab *slab)
@@ -599,6 +585,16 @@ static inline size_t slab_ksize(const struct kmem_cache *s) 
	return s->size;

}



static inline unsigned int large_kmalloc_order(const struct page *page)

{

	return page[1].flags.f & 0xff;

}



static inline size_t large_kmalloc_size(const struct page *page)

{

	return PAGE_SIZE << large_kmalloc_order(page);

}



#ifdef CONFIG_SLUB_DEBUG

void dump_unreclaimable_slab(void);

#else