Commit b1df9c57 authored by Zi Yan's avatar Zi Yan Committed by Andrew Morton
Browse files

mm/page_isolation: remove migratetype from move_freepages_block_isolate() 

Since migratetype is no longer overwritten during pageblock isolation,
moving a pageblock out of MIGRATE_ISOLATE no longer needs a new
migratetype.

Add pageblock_isolate_and_move_free_pages() and
pageblock_unisolate_and_move_free_pages() to be explicit about the page
isolation operations.  Both share the common code in
__move_freepages_block_isolate(), which is renamed from
move_freepages_block_isolate().

Add toggle_pageblock_isolate() to flip pageblock isolation bit in
__move_freepages_block_isolate().

Make set_pageblock_migratetype() only accept non MIGRATE_ISOLATE types, so
that one should use set_pageblock_isolate() to isolate pageblocks.  As a
result, move pageblock migratetype code out of __move_freepages_block().

Link: https://lkml.kernel.org/r/20250617021115.2331563-5-ziy@nvidia.com


Signed-off-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Richard Chang <richardycc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 1bc3587a

include/linux/page-isolation.h

+2 −3
Original line number Diff line number Diff line
@@ -44,10 +44,9 @@ static inline void set_pageblock_isolate(struct page *page) 
void __meminit init_pageblock_migratetype(struct page *page,

					  enum migratetype migratetype,

					  bool isolate);

void set_pageblock_migratetype(struct page *page, enum migratetype migratetype);



bool move_freepages_block_isolate(struct zone *zone, struct page *page,

				  int migratetype);

bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page);

bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page);



int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,

			     int migratetype, int flags);

mm/page_alloc.c

+63 −17
Original line number Diff line number Diff line
@@ -525,7 +525,7 @@ void clear_pfnblock_bit(const struct page *page, unsigned long pfn, 
 * @page: The page within the block of interest

 * @migratetype: migratetype to set

 */

__always_inline void set_pageblock_migratetype(struct page *page,

static void set_pageblock_migratetype(struct page *page,

				      enum migratetype migratetype)

{

	if (unlikely(page_group_by_mobility_disabled &&
@@ -534,9 +534,13 @@ __always_inline void set_pageblock_migratetype(struct page *page, 


#ifdef CONFIG_MEMORY_ISOLATION

	if (migratetype == MIGRATE_ISOLATE) {

		set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);

		VM_WARN_ONCE(1,

			"Use set_pageblock_isolate() for pageblock isolation");

		return;

	}

	VM_WARN_ONCE(get_pfnblock_bit(page, page_to_pfn(page),

				      PB_migrate_isolate),

		     "Use clear_pageblock_isolate() to unisolate pageblock");

	/* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */

#endif

	__set_pfnblock_flags_mask(page, page_to_pfn(page),
@@ -1921,8 +1925,8 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 
#endif



/*

 * Change the type of a block and move all its free pages to that

 * type's freelist.

 * Move all free pages of a block to new type's freelist. Caller needs to

 * change the block type.

 */

static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,

				  int old_mt, int new_mt)
@@ -1954,8 +1958,6 @@ static int __move_freepages_block(struct zone *zone, unsigned long start_pfn, 
		pages_moved += 1 << order;

	}



	set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);



	return pages_moved;

}
@@ -2013,11 +2015,16 @@ static int move_freepages_block(struct zone *zone, struct page *page, 
				int old_mt, int new_mt)

{

	unsigned long start_pfn;

	int res;



	if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))

		return -1;



	return __move_freepages_block(zone, start_pfn, old_mt, new_mt);

	res = __move_freepages_block(zone, start_pfn, old_mt, new_mt);

	set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);



	return res;



}



#ifdef CONFIG_MEMORY_ISOLATION
@@ -2045,11 +2052,19 @@ static unsigned long find_large_buddy(unsigned long start_pfn) 
	return start_pfn;

}



static inline void toggle_pageblock_isolate(struct page *page, bool isolate)

{

	if (isolate)

		set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);

	else

		clear_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);

}



/**

 * move_freepages_block_isolate - move free pages in block for page isolation

 * __move_freepages_block_isolate - move free pages in block for page isolation

 * @zone: the zone

 * @page: the pageblock page

 * @migratetype: migratetype to set on the pageblock

 * @isolate: to isolate the given pageblock or unisolate it

 *

 * This is similar to move_freepages_block(), but handles the special

 * case encountered in page isolation, where the block of interest
@@ -2064,10 +2079,18 @@ static unsigned long find_large_buddy(unsigned long start_pfn) 
 *

 * Returns %true if pages could be moved, %false otherwise.

 */

bool move_freepages_block_isolate(struct zone *zone, struct page *page,

				  int migratetype)

static bool __move_freepages_block_isolate(struct zone *zone,

		struct page *page, bool isolate)

{

	unsigned long start_pfn, pfn;

	int from_mt;

	int to_mt;



	if (isolate == get_pageblock_isolate(page)) {

		VM_WARN_ONCE(1, "%s a pageblock that is already in that state",

			     isolate ? "Isolate" : "Unisolate");

		return false;

	}



	if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))

		return false;
@@ -2084,7 +2107,7 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page, 


		del_page_from_free_list(buddy, zone, order,

					get_pfnblock_migratetype(buddy, pfn));

		set_pageblock_migratetype(page, migratetype);

		toggle_pageblock_isolate(page, isolate);

		split_large_buddy(zone, buddy, pfn, order, FPI_NONE);

		return true;

	}
@@ -2095,16 +2118,38 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page, 


		del_page_from_free_list(page, zone, order,

					get_pfnblock_migratetype(page, pfn));

		set_pageblock_migratetype(page, migratetype);

		toggle_pageblock_isolate(page, isolate);

		split_large_buddy(zone, page, pfn, order, FPI_NONE);

		return true;

	}

move:

	__move_freepages_block(zone, start_pfn,

			       get_pfnblock_migratetype(page, start_pfn),

			       migratetype);

	/* Use MIGRATETYPE_MASK to get non-isolate migratetype */

	if (isolate) {

		from_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),

						    MIGRATETYPE_MASK);

		to_mt = MIGRATE_ISOLATE;

	} else {

		from_mt = MIGRATE_ISOLATE;

		to_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),

						  MIGRATETYPE_MASK);

	}



	__move_freepages_block(zone, start_pfn, from_mt, to_mt);

	toggle_pageblock_isolate(pfn_to_page(start_pfn), isolate);



	return true;

}



bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page)

{

	return __move_freepages_block_isolate(zone, page, true);

}



bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page)

{

	return __move_freepages_block_isolate(zone, page, false);

}



#endif /* CONFIG_MEMORY_ISOLATION */



static void change_pageblock_range(struct page *pageblock_page,
@@ -2296,6 +2341,7 @@ try_to_claim_block(struct zone *zone, struct page *page, 
	if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||

			page_group_by_mobility_disabled) {

		__move_freepages_block(zone, start_pfn, block_type, start_type);

		set_pageblock_migratetype(pfn_to_page(start_pfn), start_type);

		return __rmqueue_smallest(zone, order, start_type);

	}

mm/page_isolation.c

+10 −11
Original line number Diff line number Diff line
@@ -188,7 +188,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ 
	unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end,

			migratetype, isol_flags);

	if (!unmovable) {

		if (!move_freepages_block_isolate(zone, page, MIGRATE_ISOLATE)) {

		if (!pageblock_isolate_and_move_free_pages(zone, page)) {

			spin_unlock_irqrestore(&zone->lock, flags);

			return -EBUSY;

		}
@@ -209,7 +209,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ 
	return -EBUSY;

}



static void unset_migratetype_isolate(struct page *page, int migratetype)

static void unset_migratetype_isolate(struct page *page)

{

	struct zone *zone;

	unsigned long flags;
@@ -262,10 +262,10 @@ static void unset_migratetype_isolate(struct page *page, int migratetype) 
		 * Isolating this block already succeeded, so this

		 * should not fail on zone boundaries.

		 */

		WARN_ON_ONCE(!move_freepages_block_isolate(zone, page, migratetype));

		WARN_ON_ONCE(!pageblock_unisolate_and_move_free_pages(zone, page));

	} else {

		set_pageblock_migratetype(page, migratetype);

		__putback_isolated_page(page, order, migratetype);

		clear_pageblock_isolate(page);

		__putback_isolated_page(page, order, get_pageblock_migratetype(page));

	}

	zone->nr_isolate_pageblock--;

out:
@@ -383,7 +383,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, 
		if (PageBuddy(page)) {

			int order = buddy_order(page);



			/* move_freepages_block_isolate() handled this */

			/* pageblock_isolate_and_move_free_pages() handled this */

			VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn);



			pfn += 1UL << order;
@@ -433,7 +433,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, 
failed:

	/* restore the original migratetype */

	if (!skip_isolation)

		unset_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype);

		unset_migratetype_isolate(pfn_to_page(isolate_pageblock));

	return -EBUSY;

}
@@ -504,7 +504,7 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, 
	ret = isolate_single_pageblock(isolate_end, flags, true,

			skip_isolation, migratetype);

	if (ret) {

		unset_migratetype_isolate(pfn_to_page(isolate_start), migratetype);

		unset_migratetype_isolate(pfn_to_page(isolate_start));

		return ret;

	}
@@ -517,8 +517,7 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, 
					start_pfn, end_pfn)) {

			undo_isolate_page_range(isolate_start, pfn, migratetype);

			unset_migratetype_isolate(

				pfn_to_page(isolate_end - pageblock_nr_pages),

				migratetype);

				pfn_to_page(isolate_end - pageblock_nr_pages));

			return -EBUSY;

		}

	}
@@ -548,7 +547,7 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, 
		page = __first_valid_page(pfn, pageblock_nr_pages);

		if (!page || !is_migrate_isolate_page(page))

			continue;

		unset_migratetype_isolate(page, migratetype);

		unset_migratetype_isolate(page);

	}

}

/*