Commit 7460b470 authored by Zi Yan's avatar Zi Yan Committed by Andrew Morton
Browse files

mm/truncate: use folio_split() in truncate operation 

Instead of splitting the large folio uniformly during truncation, try to
use buddy allocator like folio_split() at the start and the end of a
truncation range to minimize the number of resulting folios if it is
supported.  try_folio_split() is introduced to use folio_split() if
supported and it falls back to uniform split otherwise.

For example, to truncate a order-4 folio
[0, 1, 2, 3, 4, 5, ..., 15]
between [3, 10] (inclusive), folio_split() splits the folio at 3 to
[0,1], [2], [3], [4..7], [8..15] and [3], [4..7] can be dropped and
[8..15] is kept with zeros in [8..10], then another folio_split() is
done at 10, so [8..10] can be dropped.

One possible optimization is to make folio_split() to split a folio based
on a given range, like [3..10] above.  But that complicates folio_split(),
so it will be investigated when necessary.

Link: https://lkml.kernel.org/r/20250226210032.2044041-8-ziy@nvidia.com
Link: https://lkml.kernel.org/r/20250307174001.242794-8-ziy@nvidia.com


Signed-off-by: default avatarZi Yan <ziy@nvidia.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Yang Shi <yang@os.amperecomputing.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Kairui Song <kasong@tencent.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 4b94c18d

include/linux/huge_mm.h

+36 −0
Original line number Diff line number Diff line
@@ -345,6 +345,36 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 
		unsigned int new_order);

int min_order_for_split(struct folio *folio);

int split_folio_to_list(struct folio *folio, struct list_head *list);

bool uniform_split_supported(struct folio *folio, unsigned int new_order,

		bool warns);

bool non_uniform_split_supported(struct folio *folio, unsigned int new_order,

		bool warns);

int folio_split(struct folio *folio, unsigned int new_order, struct page *page,

		struct list_head *list);

/*

 * try_folio_split - try to split a @folio at @page using non uniform split.

 * @folio: folio to be split

 * @page: split to order-0 at the given page

 * @list: store the after-split folios

 *

 * Try to split a @folio at @page using non uniform split to order-0, if

 * non uniform split is not supported, fall back to uniform split.

 *

 * Return: 0: split is successful, otherwise split failed.

 */

static inline int try_folio_split(struct folio *folio, struct page *page,

		struct list_head *list)

{

	int ret = min_order_for_split(folio);



	if (ret < 0)

		return ret;



	if (!non_uniform_split_supported(folio, 0, false))

		return split_huge_page_to_list_to_order(&folio->page, list,

				ret);

	return folio_split(folio, ret, page, list);

}

static inline int split_huge_page(struct page *page)

{

	struct folio *folio = page_folio(page);
@@ -537,6 +567,12 @@ static inline int split_folio_to_list(struct folio *folio, struct list_head *lis 
	return 0;

}



static inline int try_folio_split(struct folio *folio, struct page *page,

		struct list_head *list)

{

	return 0;

}



static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {}

#define split_huge_pmd(__vma, __pmd, __address)	\

	do { } while (0)

mm/huge_memory.c

+3 −3
Original line number Diff line number Diff line
@@ -3654,7 +3654,7 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, 
	return ret;

}



static bool non_uniform_split_supported(struct folio *folio, unsigned int new_order,

bool non_uniform_split_supported(struct folio *folio, unsigned int new_order,

		bool warns)

{

	if (folio_test_anon(folio)) {
@@ -3686,7 +3686,7 @@ static bool non_uniform_split_supported(struct folio *folio, unsigned int new_or 
}



/* See comments in non_uniform_split_supported() */

static bool uniform_split_supported(struct folio *folio, unsigned int new_order,

bool uniform_split_supported(struct folio *folio, unsigned int new_order,

		bool warns)

{

	if (folio_test_anon(folio)) {
@@ -4005,7 +4005,7 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 
 *

 * After split, folio is left locked for caller.

 */

static int folio_split(struct folio *folio, unsigned int new_order,

int folio_split(struct folio *folio, unsigned int new_order,

		struct page *split_at, struct list_head *list)

{

	return __folio_split(folio, new_order, split_at, &folio->page, list,

mm/truncate.c

+36 −1
Original line number Diff line number Diff line
@@ -192,6 +192,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) 
{

	loff_t pos = folio_pos(folio);

	unsigned int offset, length;

	struct page *split_at, *split_at2;



	if (pos < start)

		offset = start - pos;
@@ -221,8 +222,42 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) 
		folio_invalidate(folio, offset, length);

	if (!folio_test_large(folio))

		return true;

	if (split_folio(folio) == 0)



	split_at = folio_page(folio, PAGE_ALIGN_DOWN(offset) / PAGE_SIZE);

	split_at2 = folio_page(folio,

			PAGE_ALIGN_DOWN(offset + length) / PAGE_SIZE);



	if (!try_folio_split(folio, split_at, NULL)) {

		/*

		 * try to split at offset + length to make sure folios within

		 * the range can be dropped, especially to avoid memory waste

		 * for shmem truncate

		 */

		struct folio *folio2 = page_folio(split_at2);



		if (!folio_try_get(folio2))

			goto no_split;



		if (!folio_test_large(folio2))

			goto out;



		if (!folio_trylock(folio2))

			goto out;



		/*

		 * make sure folio2 is large and does not change its mapping.

		 * Its split result does not matter here.

		 */

		if (folio_test_large(folio2) &&

		    folio2->mapping == folio->mapping)

			try_folio_split(folio2, split_at2, NULL);



		folio_unlock(folio2);

out:

		folio_put(folio2);

no_split:

		return true;

	}

	if (folio_test_dirty(folio))

		return false;

	truncate_inode_folio(folio->mapping, folio);