Commit dd80cfd4 authored by David Hildenbrand's avatar David Hildenbrand Committed by Andrew Morton
Browse files

mm: split folio_pte_batch() into folio_pte_batch() and folio_pte_batch_flags() 

Many users (including upcoming ones) don't really need the flags etc, and
can live with the possible overhead of a function call.

So let's provide a basic, non-inlined folio_pte_batch(), to avoid code
bloat while still providing a variant that optimizes out all flag checks
at runtime.  folio_pte_batch_flags() will get inlined into
folio_pte_batch(), optimizing out any conditionals that depend on input
flags.

folio_pte_batch() will behave like folio_pte_batch_flags() when no flags
are specified.  It's okay to add new users of folio_pte_batch_flags(), but
using folio_pte_batch() if applicable is preferred.

So, before this change, folio_pte_batch() was inlined into the C file
optimized by propagating constants within the resulting object file.

With this change, we now also have a folio_pte_batch() that is optimized
by propagating all constants.  But instead of having one instance per
object file, we have a single shared one.

In zap_present_ptes(), where we care about performance, the compiler
already seem to generate a call to a common inlined folio_pte_batch()
variant, shared with fork() code.  So calling the new non-inlined variant
should not make a difference.

While at it, drop the "addr" parameter that is unused.

Link: https://lkml.kernel.org/r/20250702104926.212243-4-david@redhat.com


Signed-off-by: default avatarDavid Hildenbrand <david@redhat.com>
Suggested-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Link: https://lore.kernel.org/linux-mm/20250503182858.5a02729fcffd6d4723afcfc2@linux-foundation.org/


Reviewed-by: default avatarOscar Salvador <osalvador@suse.de>
Reviewed-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarDev Jain <dev.jain@arm.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Byungchul Park <byungchul@sk.com>
Cc: Gregory Price <gourry@gourry.net>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Jann Horn <jannh@google.com>
Cc: Joshua Hahn <joshua.hahnjy@gmail.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 233e28e2

mm/internal.h

+8 −3
Original line number Diff line number Diff line
@@ -218,9 +218,8 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) 
}



/**

 * folio_pte_batch - detect a PTE batch for a large folio

 * folio_pte_batch_flags - detect a PTE batch for a large folio

 * @folio: The large folio to detect a PTE batch for.

 * @addr: The user virtual address the first page is mapped at.

 * @ptep: Page table pointer for the first entry.

 * @pte: Page table entry for the first page.

 * @max_nr: The maximum number of table entries to consider.
@@ -243,9 +242,12 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) 
 * must be limited by the caller so scanning cannot exceed a single VMA and

 * a single page table.

 *

 * This function will be inlined to optimize based on the input parameters;

 * consider using folio_pte_batch() instead if applicable.

 *

 * Return: the number of table entries in the batch.

 */

static inline unsigned int folio_pte_batch(struct folio *folio, unsigned long addr,

static inline unsigned int folio_pte_batch_flags(struct folio *folio,

		pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags,

		bool *any_writable, bool *any_young, bool *any_dirty)

{
@@ -293,6 +295,9 @@ static inline unsigned int folio_pte_batch(struct folio *folio, unsigned long ad 
	return min(nr, max_nr);

}



unsigned int folio_pte_batch(struct folio *folio, pte_t *ptep, pte_t pte,

		unsigned int max_nr);



/**

 * pte_move_swp_offset - Move the swap entry offset field of a swap pte

 *	 forward or backward by delta

mm/madvise.c

+2 −2
Original line number Diff line number Diff line
@@ -348,7 +348,7 @@ static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, 
{

	int max_nr = (end - addr) / PAGE_SIZE;



	return folio_pte_batch(folio, addr, ptep, pte, max_nr, 0, NULL,

	return folio_pte_batch_flags(folio, ptep, pte, max_nr, 0, NULL,

				     any_young, any_dirty);

}

mm/memory.c

+3 −5
Original line number Diff line number Diff line
@@ -995,7 +995,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma 
		if (vma_soft_dirty_enabled(src_vma))

			flags |= FPB_RESPECT_SOFT_DIRTY;



		nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,

		nr = folio_pte_batch_flags(folio, src_pte, pte, max_nr, flags,

					   &any_writable, NULL, NULL);

		folio_ref_add(folio, nr);

		if (folio_test_anon(folio)) {
@@ -1564,9 +1564,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, 
	 * by keeping the batching logic separate.

	 */

	if (unlikely(folio_test_large(folio) && max_nr != 1)) {

		nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, 0,

				     NULL, NULL, NULL);



		nr = folio_pte_batch(folio, pte, ptent, max_nr);

		zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,

				       addr, details, rss, force_flush,

				       force_break, any_skipped);

mm/mempolicy.c

+1 −2
Original line number Diff line number Diff line
@@ -711,8 +711,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, 
		if (!folio || folio_is_zone_device(folio))

			continue;

		if (folio_test_large(folio) && max_nr != 1)

			nr = folio_pte_batch(folio, addr, pte, ptent,

					     max_nr, 0, NULL, NULL, NULL);

			nr = folio_pte_batch(folio, pte, ptent, max_nr);

		/*

		 * vm_normal_folio() filters out zero pages, but there might

		 * still be reserved folios to skip, perhaps in a VDSO.

mm/mlock.c

+1 −2
Original line number Diff line number Diff line
@@ -313,8 +313,7 @@ static inline unsigned int folio_mlock_step(struct folio *folio, 
	if (!folio_test_large(folio))

		return 1;



	return folio_pte_batch(folio, addr, pte, ptent, count, 0, NULL,

			       NULL, NULL);

	return folio_pte_batch(folio, pte, ptent, count);

}



static inline bool allow_mlock_munlock(struct folio *folio,