Commit e66d7a4f authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm: convert FPB_IGNORE_* into FPB_RESPECT_* 

Patch series "mm: folio_pte_batch() improvements", v2.

Ever since we added folio_pte_batch() for fork() + munmap() purposes, a
lot more users appeared (and more are being proposed), and more
functionality was added.

Most of the users only need basic functionality, and could benefit from a
non-inlined version.

So let's clean up folio_pte_batch() and split it into a basic
folio_pte_batch() (no flags) and a more advanced folio_pte_batch_ext(). 
Using either variant will now look much cleaner.

This series will likely conflict with some changes in some (old+new)
folio_pte_batch() users, but conflicts should be trivial to resolve.


This patch (of 4):

Respecting these PTE bits is the exception, so let's invert the meaning.

With this change, most callers don't have to pass any flags.  This is a
preparation for splitting folio_pte_batch() into a non-inlined variant
that doesn't consume any flags.

Long-term, we want folio_pte_batch() to probably ignore most common PTE
bits (e.g., write/dirty/young/soft-dirty) that are not relevant for most
page table walkers: uffd-wp and protnone might be bits to consider in the
future.  Only walkers that care about them can opt-in to respect them.

No functional change intended.

Link: https://lkml.kernel.org/r/20250702104926.212243-2-david@redhat.com


Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarLance Yang <lance.yang@linux.dev>
Reviewed-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarOscar Salvador <osalvador@suse.de>
Reviewed-by: default avatarDev Jain <dev.jain@arm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Jann Horn <jannh@google.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 77657948

mm/internal.h

+8 −8
Original line number Diff line number Diff line
@@ -202,17 +202,17 @@ static inline void vma_close(struct vm_area_struct *vma) 
/* Flags for folio_pte_batch(). */

typedef int __bitwise fpb_t;



/* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */

#define FPB_IGNORE_DIRTY		((__force fpb_t)BIT(0))

/* Compare PTEs respecting the dirty bit. */

#define FPB_RESPECT_DIRTY		((__force fpb_t)BIT(0))



/* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */

#define FPB_IGNORE_SOFT_DIRTY		((__force fpb_t)BIT(1))

/* Compare PTEs respecting the soft-dirty bit. */

#define FPB_RESPECT_SOFT_DIRTY		((__force fpb_t)BIT(1))



static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)

{

	if (flags & FPB_IGNORE_DIRTY)

	if (!(flags & FPB_RESPECT_DIRTY))

		pte = pte_mkclean(pte);

	if (likely(flags & FPB_IGNORE_SOFT_DIRTY))

	if (likely(!(flags & FPB_RESPECT_SOFT_DIRTY)))

		pte = pte_clear_soft_dirty(pte);

	return pte_wrprotect(pte_mkold(pte));

}
@@ -236,8 +236,8 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) 
 * pages of the same large folio.

 *

 * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,

 * the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and

 * soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY).

 * the accessed bit, writable bit, dirty bit (unless FPB_RESPECT_DIRTY is set)

 * and soft-dirty bit (unless FPB_RESPECT_SOFT_DIRTY is set).

 *

 * start_ptep must map any page of the folio. max_nr must be at least one and

 * must be limited by the caller so scanning cannot exceed a single page table.

mm/madvise.c

+1 −2
Original line number Diff line number Diff line
@@ -346,10 +346,9 @@ static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, 
					  pte_t pte, bool *any_young,

					  bool *any_dirty)

{

	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;

	int max_nr = (end - addr) / PAGE_SIZE;



	return folio_pte_batch(folio, addr, ptep, pte, max_nr, fpb_flags, NULL,

	return folio_pte_batch(folio, addr, ptep, pte, max_nr, 0, NULL,

			       any_young, any_dirty);

}

mm/memory.c

+5 −6
Original line number Diff line number Diff line
@@ -990,10 +990,10 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma 
	 * by keeping the batching logic separate.

	 */

	if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {

		if (src_vma->vm_flags & VM_SHARED)

			flags |= FPB_IGNORE_DIRTY;

		if (!vma_soft_dirty_enabled(src_vma))

			flags |= FPB_IGNORE_SOFT_DIRTY;

		if (!(src_vma->vm_flags & VM_SHARED))

			flags |= FPB_RESPECT_DIRTY;

		if (vma_soft_dirty_enabled(src_vma))

			flags |= FPB_RESPECT_SOFT_DIRTY;



		nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,

				     &any_writable, NULL, NULL);
@@ -1535,7 +1535,6 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, 
		struct zap_details *details, int *rss, bool *force_flush,

		bool *force_break, bool *any_skipped)

{

	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;

	struct mm_struct *mm = tlb->mm;

	struct folio *folio;

	struct page *page;
@@ -1565,7 +1564,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, 
	 * by keeping the batching logic separate.

	 */

	if (unlikely(folio_test_large(folio) && max_nr != 1)) {

		nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags,

		nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, 0,

				     NULL, NULL, NULL);



		zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,

mm/mempolicy.c

+1 −3
Original line number Diff line number Diff line
@@ -675,7 +675,6 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) 
static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,

			unsigned long end, struct mm_walk *walk)

{

	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;

	struct vm_area_struct *vma = walk->vma;

	struct folio *folio;

	struct queue_pages *qp = walk->private;
@@ -713,8 +712,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, 
			continue;

		if (folio_test_large(folio) && max_nr != 1)

			nr = folio_pte_batch(folio, addr, pte, ptent,

					     max_nr, fpb_flags,

					     NULL, NULL, NULL);

					     max_nr, 0, NULL, NULL, NULL);

		/*

		 * vm_normal_folio() filters out zero pages, but there might

		 * still be reserved folios to skip, perhaps in a VDSO.

mm/mlock.c

+1 −2
Original line number Diff line number Diff line
@@ -307,14 +307,13 @@ void munlock_folio(struct folio *folio) 
static inline unsigned int folio_mlock_step(struct folio *folio,

		pte_t *pte, unsigned long addr, unsigned long end)

{

	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;

	unsigned int count = (end - addr) >> PAGE_SHIFT;

	pte_t ptent = ptep_get(pte);



	if (!folio_test_large(folio))

		return 1;



	return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,

	return folio_pte_batch(folio, addr, pte, ptent, count, 0, NULL,

			       NULL, NULL);

}