Commit e904bce2 authored by Zi Yan's avatar Zi Yan Committed by Andrew Morton
Browse files

mm/page_isolation: make page isolation a standalone bit 

During page isolation, the original migratetype is overwritten, since
MIGRATE_* are enums and stored in pageblock bitmaps.  Change
MIGRATE_ISOLATE to be stored a standalone bit, PB_migrate_isolate, like
PB_compact_skip, so that migratetype is not lost during pageblock
isolation.

Link: https://lkml.kernel.org/r/20250617021115.2331563-3-ziy@nvidia.com


Signed-off-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Richard Chang <richardycc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 42f46ed9

include/linux/mmzone.h

+3 −0
Original line number Diff line number Diff line
@@ -79,6 +79,9 @@ enum migratetype { 
	 * __free_pageblock_cma() function.

	 */

	MIGRATE_CMA,

	__MIGRATE_TYPE_END = MIGRATE_CMA,

#else

	__MIGRATE_TYPE_END = MIGRATE_HIGHATOMIC,

#endif

#ifdef CONFIG_MEMORY_ISOLATION

	MIGRATE_ISOLATE,	/* can't allocate from here */

include/linux/page-isolation.h

+16 −0
Original line number Diff line number Diff line
@@ -11,6 +11,12 @@ static inline bool is_migrate_isolate(int migratetype) 
{

	return migratetype == MIGRATE_ISOLATE;

}

#define get_pageblock_isolate(page) \

	get_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)

#define clear_pageblock_isolate(page) \

	clear_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)

#define set_pageblock_isolate(page) \

	set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)

#else

static inline bool is_migrate_isolate_page(struct page *page)

{
@@ -20,6 +26,16 @@ static inline bool is_migrate_isolate(int migratetype) 
{

	return false;

}

static inline bool get_pageblock_isolate(struct page *page)

{

	return false;

}

static inline void clear_pageblock_isolate(struct page *page)

{

}

static inline void set_pageblock_isolate(struct page *page)

{

}

#endif



#define MEMORY_OFFLINE	0x1

include/linux/pageblock-flags.h

+14 −0
Original line number Diff line number Diff line
@@ -21,6 +21,13 @@ enum pageblock_bits { 
			/* 3 bits required for migrate types */

	PB_compact_skip,/* If set the block is skipped by compaction */



#ifdef CONFIG_MEMORY_ISOLATION

	/*

	 * Pageblock isolation is represented with a separate bit, so that

	 * the migratetype of a block is not overwritten by isolation.

	 */

	PB_migrate_isolate, /* If set the block is isolated */

#endif

	/*

	 * Assume the bits will always align on a word. If this assumption

	 * changes then get/set pageblock needs updating.
@@ -32,6 +39,13 @@ enum pageblock_bits { 


#define MIGRATETYPE_MASK ((1UL << (PB_migrate_end + 1)) - 1)



#ifdef CONFIG_MEMORY_ISOLATION

#define MIGRATETYPE_AND_ISO_MASK \

	(((1UL << (PB_migrate_end + 1)) - 1) | BIT(PB_migrate_isolate))

#else

#define MIGRATETYPE_AND_ISO_MASK MIGRATETYPE_MASK

#endif



#if defined(CONFIG_HUGETLB_PAGE)



#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE

mm/page_alloc.c

+24 −3
Original line number Diff line number Diff line
@@ -365,8 +365,12 @@ get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn, 
	unsigned long *bitmap;

	unsigned long word_bitidx;



#ifdef CONFIG_MEMORY_ISOLATION

	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);

#else

	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

	BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));

#endif

	BUILD_BUG_ON(__MIGRATE_TYPE_END >= (1 << PB_migratetype_bits));

	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);



	bitmap = get_pageblock_bitmap(page, pfn);
@@ -439,7 +443,16 @@ bool get_pfnblock_bit(const struct page *page, unsigned long pfn, 
__always_inline enum migratetype

get_pfnblock_migratetype(const struct page *page, unsigned long pfn)

{

	return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);

	unsigned long mask = MIGRATETYPE_AND_ISO_MASK;

	unsigned long flags;



	flags = __get_pfnblock_flags_mask(page, pfn, mask);



#ifdef CONFIG_MEMORY_ISOLATION

	if (flags & BIT(PB_migrate_isolate))

		return MIGRATE_ISOLATE;

#endif

	return flags & MIGRATETYPE_MASK;

}



/**
@@ -519,8 +532,16 @@ __always_inline void set_pageblock_migratetype(struct page *page, 
		     migratetype < MIGRATE_PCPTYPES))

		migratetype = MIGRATE_UNMOVABLE;



#ifdef CONFIG_MEMORY_ISOLATION

	if (migratetype == MIGRATE_ISOLATE) {

		set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);

		return;

	}

	/* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */

#endif

	__set_pfnblock_flags_mask(page, page_to_pfn(page),

				  (unsigned long)migratetype, MIGRATETYPE_MASK);

				  (unsigned long)migratetype,

				  MIGRATETYPE_AND_ISO_MASK);

}



#ifdef CONFIG_DEBUG_VM