Commit 45fec1e5 authored by Qi Zheng's avatar Qi Zheng Committed by Andrew Morton
Browse files

mm: skip over all consecutive none ptes in do_zap_pte_range() 

Skip over all consecutive none ptes in do_zap_pte_range(), which helps
optimize away need_resched() + force_break + incremental pte/addr
increments etc.

Link: https://lkml.kernel.org/r/8ecffbf990afd1c8ccc195a2ec321d55f0923908.1733305182.git.zhengqi.arch@bytedance.com


Signed-off-by: default avatarQi Zheng <zhengqi.arch@bytedance.com>
Suggested-by: default avatarDavid Hildenbrand <david@redhat.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Peter Xu <peterx@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Cc: Zach O'Keefe <zokeefe@google.com>
Cc: Dan Carpenter <dan.carpenter@linaro.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 117cdb05

mm/memory.c

+20 −7
Original line number Diff line number Diff line
@@ -1665,17 +1665,30 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, 
{

	pte_t ptent = ptep_get(pte);

	int max_nr = (end - addr) / PAGE_SIZE;

	int nr = 0;



	if (pte_none(ptent))

		return 1;

	/* Skip all consecutive none ptes */

	if (pte_none(ptent)) {

		for (nr = 1; nr < max_nr; nr++) {

			ptent = ptep_get(pte + nr);

			if (!pte_none(ptent))

				break;

		}

		max_nr -= nr;

		if (!max_nr)

			return nr;

		pte += nr;

		addr += nr * PAGE_SIZE;

	}



	if (pte_present(ptent))

		return zap_present_ptes(tlb, vma, pte, ptent, max_nr,

					addr, details, rss, force_flush,

					force_break);



	return zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr,

		nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr,

				       details, rss, force_flush, force_break);

	else

		nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr,

					  details, rss);



	return nr;

}



static unsigned long zap_pte_range(struct mmu_gather *tlb,