Commit 44562c71 authored by Ryan Roberts's avatar Ryan Roberts Committed by Will Deacon
Browse files

mm/vmalloc: Enter lazy mmu mode while manipulating vmalloc ptes 



Wrap vmalloc's pte table manipulation loops with
arch_enter_lazy_mmu_mode() / arch_leave_lazy_mmu_mode(). This provides
the arch code with the opportunity to optimize the pte manipulations.

Note that vmap_pfn() already uses lazy mmu mode since it delegates to
apply_to_page_range() which enters lazy mmu mode for both user and
kernel mappings.

These hooks will shortly be used by arm64 to improve vmalloc
performance.

Reviewed-by: default avatarUladzislau Rezki (Sony) <urezki@gmail.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Reviewed-by: default avatarAnshuman Khandual <anshuman.khandual@arm.com>
Signed-off-by: default avatarRyan Roberts <ryan.roberts@arm.com>
Tested-by: default avatarLuiz Capitulino <luizcap@redhat.com>
Link: https://lore.kernel.org/r/20250422081822.1836315-11-ryan.roberts@arm.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 06fc959f

mm/vmalloc.c

+14 −0
Original line number Diff line number Diff line
@@ -104,6 +104,9 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
	pte = pte_alloc_kernel_track(pmd, addr, mask);

	if (!pte)

		return -ENOMEM;



	arch_enter_lazy_mmu_mode();



	do {

		if (unlikely(!pte_none(ptep_get(pte)))) {

			if (pfn_valid(pfn)) {
@@ -127,6 +130,8 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));

		pfn++;

	} while (pte += PFN_DOWN(size), addr += size, addr != end);



	arch_leave_lazy_mmu_mode();

	*mask |= PGTBL_PTE_MODIFIED;

	return 0;

}
@@ -354,6 +359,8 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
	unsigned long size = PAGE_SIZE;



	pte = pte_offset_kernel(pmd, addr);

	arch_enter_lazy_mmu_mode();



	do {

#ifdef CONFIG_HUGETLB_PAGE

		size = arch_vmap_pte_range_unmap_size(addr, pte);
@@ -370,6 +377,8 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
			ptent = ptep_get_and_clear(&init_mm, addr, pte);

		WARN_ON(!pte_none(ptent) && !pte_present(ptent));

	} while (pte += (size >> PAGE_SHIFT), addr += size, addr != end);



	arch_leave_lazy_mmu_mode();

	*mask |= PGTBL_PTE_MODIFIED;

}
@@ -515,6 +524,9 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 
	pte = pte_alloc_kernel_track(pmd, addr, mask);

	if (!pte)

		return -ENOMEM;



	arch_enter_lazy_mmu_mode();



	do {

		struct page *page = pages[*nr];
@@ -528,6 +540,8 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 
		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));

		(*nr)++;

	} while (pte++, addr += PAGE_SIZE, addr != end);



	arch_leave_lazy_mmu_mode();

	*mask |= PGTBL_PTE_MODIFIED;

	return 0;

}