Commit e0c13267 authored by Kefeng Wang's avatar Kefeng Wang Committed by Andrew Morton
Browse files

mm: page_alloc: add alloc_contig_frozen_{range,pages}() 

In order to allocate given range of pages or allocate compound pages
without incrementing their refcount, adding two new helper
alloc_contig_frozen_{range,pages}() which may be beneficial to some users
(eg hugetlb).

The new alloc_contig_{range,pages} only take !__GFP_COMP gfp now, and the
free_contig_range() is refactored to only free non-compound pages, the
only caller to free compound pages in cma_free_folio() is changed
accordingly, and the free_contig_frozen_range() is provided to match the
alloc_contig_frozen_range(), which is used to free frozen pages.

Link: https://lkml.kernel.org/r/20260109093136.1491549-5-wangkefeng.wang@huawei.com


Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarSidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Claudiu Beznea <claudiu.beznea.uj@bp.renesas.com>
Cc: Mark Brown <broonie@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 6c08cc64

include/linux/gfp.h

+21 −31
Original line number Diff line number Diff line
@@ -436,39 +436,29 @@ typedef unsigned int __bitwise acr_flags_t; 
#define ACR_FLAGS_CMA ((__force acr_flags_t)BIT(0)) // allocate for CMA



/* The below functions must be run on a range from a single zone. */

extern int alloc_contig_range_noprof(unsigned long start, unsigned long end,

int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end,

		acr_flags_t alloc_flags, gfp_t gfp_mask);

#define alloc_contig_range(...)			alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__))

#define alloc_contig_frozen_range(...)	\

	alloc_hooks(alloc_contig_frozen_range_noprof(__VA_ARGS__))



extern struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,

					      int nid, nodemask_t *nodemask);

#define alloc_contig_pages(...)			alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__))



void free_contig_range(unsigned long pfn, unsigned long nr_pages);

#endif



#ifdef CONFIG_CONTIG_ALLOC

static inline struct folio *folio_alloc_gigantic_noprof(int order, gfp_t gfp,

							int nid, nodemask_t *node)

{

	struct page *page;

int alloc_contig_range_noprof(unsigned long start, unsigned long end,

		acr_flags_t alloc_flags, gfp_t gfp_mask);

#define alloc_contig_range(...)	\

	alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__))



	if (WARN_ON(!order || !(gfp & __GFP_COMP)))

		return NULL;

struct page *alloc_contig_frozen_pages_noprof(unsigned long nr_pages,

		gfp_t gfp_mask, int nid, nodemask_t *nodemask);

#define alloc_contig_frozen_pages(...) \

	alloc_hooks(alloc_contig_frozen_pages_noprof(__VA_ARGS__))



	page = alloc_contig_pages_noprof(1 << order, gfp, nid, node);

struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,

		int nid, nodemask_t *nodemask);

#define alloc_contig_pages(...)	\

	alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__))



	return page ? page_folio(page) : NULL;

}

#else

static inline struct folio *folio_alloc_gigantic_noprof(int order, gfp_t gfp,

							int nid, nodemask_t *node)

{

	return NULL;

}

void free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages);

void free_contig_range(unsigned long pfn, unsigned long nr_pages);

#endif

/* This should be paired with folio_put() rather than free_contig_range(). */

#define folio_alloc_gigantic(...) alloc_hooks(folio_alloc_gigantic_noprof(__VA_ARGS__))



DEFINE_FREE(free_page, void *, free_page((unsigned long)_T))

mm/cma.c

+7 −2
Original line number Diff line number Diff line
@@ -836,7 +836,7 @@ static int cma_range_alloc(struct cma *cma, struct cma_memrange *cmr, 
		spin_unlock_irq(&cma->lock);



		mutex_lock(&cma->alloc_mutex);

		ret = alloc_contig_range(pfn, pfn + count, ACR_FLAGS_CMA, gfp);

		ret = alloc_contig_frozen_range(pfn, pfn + count, ACR_FLAGS_CMA, gfp);

		mutex_unlock(&cma->alloc_mutex);

		if (!ret)

			break;
@@ -904,6 +904,7 @@ static struct page *__cma_alloc(struct cma *cma, unsigned long count, 
	trace_cma_alloc_finish(name, page ? page_to_pfn(page) : 0,

			       page, count, align, ret);

	if (page) {

		set_pages_refcounted(page, count);

		count_vm_event(CMA_ALLOC_SUCCESS);

		cma_sysfs_account_success_pages(cma, count);

	} else {
@@ -983,7 +984,11 @@ bool cma_release(struct cma *cma, const struct page *pages, 
		return false;

	}



	if (PageHead(pages))

		__free_pages((struct page *)pages, compound_order(pages));

	else

		free_contig_range(pfn, count);



	cma_clear_bitmap(cma, cmr, pfn, count);

	cma_sysfs_account_release_pages(cma, count);

	trace_cma_release(cma->name, pfn, pages, count);

mm/hugetlb.c

+7 −2
Original line number Diff line number Diff line
@@ -1428,12 +1428,17 @@ static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask, 
retry:

	folio = hugetlb_cma_alloc_folio(order, gfp_mask, nid, nodemask);

	if (!folio) {

		struct page *page;



		if (hugetlb_cma_exclusive_alloc())

			return NULL;



		folio = folio_alloc_gigantic(order, gfp_mask, nid, nodemask);

		if (!folio)

		page = alloc_contig_frozen_pages(1 << order, gfp_mask, nid, nodemask);

		if (!page)

			return NULL;



		set_page_refcounted(page);

		folio = page_folio(page);

	}



	if (folio_ref_freeze(folio, 1))

mm/internal.h

+13 −0
Original line number Diff line number Diff line
@@ -580,6 +580,19 @@ static inline void set_page_refcounted(struct page *page) 
	set_page_count(page, 1);

}



static inline void set_pages_refcounted(struct page *page, unsigned long nr_pages)

{

	unsigned long pfn = page_to_pfn(page);



	if (PageHead(page)) {

		set_page_refcounted(page);

		return;

	}



	for (; nr_pages--; pfn++)

		set_page_refcounted(pfn_to_page(pfn));

}



/*

 * Return true if a folio needs ->release_folio() calling upon it.

 */

mm/page_alloc.c

+136 −50
Original line number Diff line number Diff line
@@ -6889,7 +6889,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, 
	return (ret < 0) ? ret : 0;

}



static void split_free_pages(struct list_head *list, gfp_t gfp_mask)

static void split_free_frozen_pages(struct list_head *list, gfp_t gfp_mask)

{

	int order;
@@ -6901,11 +6901,10 @@ static void split_free_pages(struct list_head *list, gfp_t gfp_mask) 
			int i;



			post_alloc_hook(page, order, gfp_mask);

			set_page_refcounted(page);

			if (!order)

				continue;



			split_page(page, order);

			__split_page(page, order);



			/* Add all subpages to the order-0 head, in sequence. */

			list_del(&page->lru);
@@ -6949,8 +6948,14 @@ static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask) 
	return 0;

}



static void __free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages)

{

	for (; nr_pages--; pfn++)

		free_frozen_pages(pfn_to_page(pfn), 0);

}



/**

 * alloc_contig_range() -- tries to allocate given range of pages

 * alloc_contig_frozen_range() -- tries to allocate given range of frozen pages

 * @start:	start PFN to allocate

 * @end:	one-past-the-last PFN to allocate

 * @alloc_flags:	allocation information
@@ -6965,11 +6970,14 @@ static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask) 
 * pageblocks in the range.  Once isolated, the pageblocks should not

 * be modified by others.

 *

 * Return: zero on success or negative error code.  On success all

 * pages which PFN is in [start, end) are allocated for the caller and

 * need to be freed with free_contig_range().

 * All frozen pages which PFN is in [start, end) are allocated for the

 * caller, and they could be freed with free_contig_frozen_range(),

 * free_frozen_pages() also could be used to free compound frozen pages

 * directly.

 *

 * Return: zero on success or negative error code.

 */

int alloc_contig_range_noprof(unsigned long start, unsigned long end,

int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end,

		acr_flags_t alloc_flags, gfp_t gfp_mask)

{

	const unsigned int order = ilog2(end - start);
@@ -7086,19 +7094,18 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, 
	}



	if (!(gfp_mask & __GFP_COMP)) {

		split_free_pages(cc.freepages, gfp_mask);

		split_free_frozen_pages(cc.freepages, gfp_mask);



		/* Free head and tail (if any) */

		if (start != outer_start)

			free_contig_range(outer_start, start - outer_start);

			__free_contig_frozen_range(outer_start, start - outer_start);

		if (end != outer_end)

			free_contig_range(end, outer_end - end);

			__free_contig_frozen_range(end, outer_end - end);

	} else if (start == outer_start && end == outer_end && is_power_of_2(end - start)) {

		struct page *head = pfn_to_page(start);



		check_new_pages(head, order);

		prep_new_page(head, order, gfp_mask, 0);

		set_page_refcounted(head);

	} else {

		ret = -EINVAL;

		WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n",
@@ -7108,16 +7115,40 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, 
	undo_isolate_page_range(start, end);

	return ret;

}

EXPORT_SYMBOL(alloc_contig_range_noprof);

EXPORT_SYMBOL(alloc_contig_frozen_range_noprof);



static int __alloc_contig_pages(unsigned long start_pfn,

				unsigned long nr_pages, gfp_t gfp_mask)

/**

 * alloc_contig_range() -- tries to allocate given range of pages

 * @start:	start PFN to allocate

 * @end:	one-past-the-last PFN to allocate

 * @alloc_flags:	allocation information

 * @gfp_mask:	GFP mask.

 *

 * This routine is a wrapper around alloc_contig_frozen_range(), it can't

 * be used to allocate compound pages, the refcount of each allocated page

 * will be set to one.

 *

 * All pages which PFN is in [start, end) are allocated for the caller,

 * and should be freed with free_contig_range() or by manually calling

 * __free_page() on each allocated page.

 *

 * Return: zero on success or negative error code.

 */

int alloc_contig_range_noprof(unsigned long start, unsigned long end,

			      acr_flags_t alloc_flags, gfp_t gfp_mask)

{

	unsigned long end_pfn = start_pfn + nr_pages;

	int ret;



	return alloc_contig_range_noprof(start_pfn, end_pfn, ACR_FLAGS_NONE,

					 gfp_mask);

	if (WARN_ON(gfp_mask & __GFP_COMP))

		return -EINVAL;



	ret = alloc_contig_frozen_range_noprof(start, end, alloc_flags, gfp_mask);

	if (!ret)

		set_pages_refcounted(pfn_to_page(start), end - start);



	return ret;

}

EXPORT_SYMBOL(alloc_contig_range_noprof);



static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,

				   unsigned long nr_pages, bool skip_hugetlb,
@@ -7186,7 +7217,7 @@ static bool zone_spans_last_pfn(const struct zone *zone, 
}



/**

 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages

 * alloc_contig_frozen_pages() -- tries to find and allocate contiguous range of frozen pages

 * @nr_pages:	Number of contiguous pages to allocate

 * @gfp_mask:	GFP mask. Node/zone/placement hints limit the search; only some

 *		action and reclaim modifiers are supported. Reclaim modifiers
@@ -7194,22 +7225,25 @@ static bool zone_spans_last_pfn(const struct zone *zone, 
 * @nid:	Target node

 * @nodemask:	Mask for other possible nodes

 *

 * This routine is a wrapper around alloc_contig_range(). It scans over zones

 * on an applicable zonelist to find a contiguous pfn range which can then be

 * tried for allocation with alloc_contig_range(). This routine is intended

 * for allocation requests which can not be fulfilled with the buddy allocator.

 * This routine is a wrapper around alloc_contig_frozen_range(). It scans over

 * zones on an applicable zonelist to find a contiguous pfn range which can then

 * be tried for allocation with alloc_contig_frozen_range(). This routine is

 * intended for allocation requests which can not be fulfilled with the buddy

 * allocator.

 *

 * The allocated memory is always aligned to a page boundary. If nr_pages is a

 * power of two, then allocated range is also guaranteed to be aligned to same

 * nr_pages (e.g. 1GB request would be aligned to 1GB).

 *

 * Allocated pages can be freed with free_contig_range() or by manually calling

 * __free_page() on each allocated page.

 * Allocated frozen pages need be freed with free_contig_frozen_range(),

 * or by manually calling free_frozen_pages() on each allocated frozen

 * non-compound page, for compound frozen pages could be freed with

 * free_frozen_pages() directly.

 *

 * Return: pointer to contiguous pages on success, or NULL if not successful.

 * Return: pointer to contiguous frozen pages on success, or NULL if not successful.

 */

struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,

				 int nid, nodemask_t *nodemask)

struct page *alloc_contig_frozen_pages_noprof(unsigned long nr_pages,

		gfp_t gfp_mask, int nid, nodemask_t *nodemask)

{

	unsigned long ret, pfn, flags;

	struct zonelist *zonelist;
@@ -7231,13 +7265,15 @@ struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 
						   &skipped_hugetlb)) {

				/*

				 * We release the zone lock here because

				 * alloc_contig_range() will also lock the zone

				 * at some point. If there's an allocation

				 * spinning on this lock, it may win the race

				 * and cause alloc_contig_range() to fail...

				 * alloc_contig_frozen_range() will also lock

				 * the zone at some point. If there's an

				 * allocation spinning on this lock, it may

				 * win the race and cause allocation to fail.

				 */

				spin_unlock_irqrestore(&zone->lock, flags);

				ret = __alloc_contig_pages(pfn, nr_pages,

				ret = alloc_contig_frozen_range_noprof(pfn,

							pfn + nr_pages,

							ACR_FLAGS_NONE,

							gfp_mask);

				if (!ret)

					return pfn_to_page(pfn);
@@ -7260,30 +7296,80 @@ struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 
	}

	return NULL;

}

EXPORT_SYMBOL(alloc_contig_frozen_pages_noprof);



void free_contig_range(unsigned long pfn, unsigned long nr_pages)

/**

 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages

 * @nr_pages:	Number of contiguous pages to allocate

 * @gfp_mask:	GFP mask.

 * @nid:	Target node

 * @nodemask:	Mask for other possible nodes

 *

 * This routine is a wrapper around alloc_contig_frozen_pages(), it can't

 * be used to allocate compound pages, the refcount of each allocated page

 * will be set to one.

 *

 * Allocated pages can be freed with free_contig_range() or by manually

 * calling __free_page() on each allocated page.

 *

 * Return: pointer to contiguous pages on success, or NULL if not successful.

 */

struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,

		int nid, nodemask_t *nodemask)

{

	unsigned long count = 0;

	struct folio *folio = pfn_folio(pfn);

	struct page *page;



	if (folio_test_large(folio)) {

		int expected = folio_nr_pages(folio);

	if (WARN_ON(gfp_mask & __GFP_COMP))

		return NULL;



		if (nr_pages == expected)

			folio_put(folio);

		else

			WARN(true, "PFN %lu: nr_pages %lu != expected %d\n",

			     pfn, nr_pages, expected);

		return;

	page = alloc_contig_frozen_pages_noprof(nr_pages, gfp_mask, nid,

						nodemask);

	if (page)

		set_pages_refcounted(page, nr_pages);



	return page;

}

EXPORT_SYMBOL(alloc_contig_pages_noprof);



	for (; nr_pages--; pfn++) {

		struct page *page = pfn_to_page(pfn);

/**

 * free_contig_frozen_range() -- free the contiguous range of frozen pages

 * @pfn:	start PFN to free

 * @nr_pages:	Number of contiguous frozen pages to free

 *

 * This can be used to free the allocated compound/non-compound frozen pages.

 */

void free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages)

{

	struct page *first_page = pfn_to_page(pfn);

	const unsigned int order = ilog2(nr_pages);



		count += page_count(page) != 1;

		__free_page(page);

	if (WARN_ON_ONCE(first_page != compound_head(first_page)))

		return;



	if (PageHead(first_page)) {

		WARN_ON_ONCE(order != compound_order(first_page));

		free_frozen_pages(first_page, order);

		return;

	}



	__free_contig_frozen_range(pfn, nr_pages);

}

	WARN(count != 0, "%lu pages are still in use!\n", count);

EXPORT_SYMBOL(free_contig_frozen_range);



/**

 * free_contig_range() -- free the contiguous range of pages

 * @pfn:	start PFN to free

 * @nr_pages:	Number of contiguous pages to free

 *

 * This can be only used to free the allocated non-compound pages.

 */

void free_contig_range(unsigned long pfn, unsigned long nr_pages)

{

	if (WARN_ON_ONCE(PageHead(pfn_to_page(pfn))))

		return;



	for (; nr_pages--; pfn++)

		__free_page(pfn_to_page(pfn));

}

EXPORT_SYMBOL(free_contig_range);

#endif /* CONFIG_CONTIG_ALLOC */