Commit 1ced09e0 authored by Dev Jain's avatar Dev Jain Committed by Andrew Morton
Browse files

mm: allocate THP on hugezeropage wp-fault 

Introduce do_huge_zero_wp_pmd() to handle wp-fault on a hugezeropage and
replace it with a PMD-mapped THP.  Remember to flush TLB entry
corresponding to the hugezeropage.  In case of failure, fallback to
splitting the PMD.

Link: https://lkml.kernel.org/r/20241008061746.285961-3-dev.jain@arm.com


Signed-off-by: default avatarDev Jain <dev.jain@arm.com>
Acked-by: default avatarDavid Hildenbrand <david@redhat.com>
Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@kernel.org>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoph Lameter <cl@gentwo.org>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Cc: Yang Shi <yang@os.amperecomputing.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent ebcfc63d

mm/huge_memory.c

+40 −1
Original line number Diff line number Diff line
@@ -1778,6 +1778,38 @@ void huge_pmd_set_accessed(struct vm_fault *vmf) 
	spin_unlock(vmf->ptl);

}



static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf)

{

	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;

	struct vm_area_struct *vma = vmf->vma;

	struct mmu_notifier_range range;

	struct folio *folio;

	vm_fault_t ret = 0;



	folio = vma_alloc_anon_folio_pmd(vma, vmf->address);

	if (unlikely(!folio))

		return VM_FAULT_FALLBACK;



	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr,

				haddr + HPAGE_PMD_SIZE);

	mmu_notifier_invalidate_range_start(&range);

	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);

	if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd)))

		goto release;

	ret = check_stable_address_space(vma->vm_mm);

	if (ret)

		goto release;

	(void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd);

	map_anon_folio_pmd(folio, vmf->pmd, vma, haddr);

	goto unlock;

release:

	folio_put(folio);

unlock:

	spin_unlock(vmf->ptl);

	mmu_notifier_invalidate_range_end(&range);

	return ret;

}



vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)

{

	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
@@ -1790,8 +1822,15 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 
	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);

	VM_BUG_ON_VMA(!vma->anon_vma, vma);



	if (is_huge_zero_pmd(orig_pmd))

	if (is_huge_zero_pmd(orig_pmd)) {

		vm_fault_t ret = do_huge_zero_wp_pmd(vmf);



		if (!(ret & VM_FAULT_FALLBACK))

			return ret;



		/* Fallback to splitting PMD if THP cannot be allocated */

		goto fallback;

	}



	spin_lock(vmf->ptl);