Commit db6b435d authored by Qi Zheng's avatar Qi Zheng Committed by Andrew Morton
Browse files

mm: pgtable: introduce pagetable_dtor() 

The pagetable_p*_dtor() are exactly the same except for the handling of
ptlock.  If we make ptlock_free() handle the case where ptdesc->ptl is
NULL and remove VM_BUG_ON_PAGE() from pmd_ptlock_free(), we can unify
pagetable_p*_dtor() into one function.  Let's introduce pagetable_dtor()
to do this.

Later, pagetable_dtor() will be moved to tlb_remove_ptdesc(), so that
ptlock and page table pages can be freed together (regardless of whether
RCU is used).  This prevents the use-after-free problem where the ptlock
is freed immediately but the page table pages is freed later via RCU.

Link: https://lkml.kernel.org/r/47f44fff9dc68d9d9e9a0d6c036df275f820598a.1736317725.git.zhengqi.arch@bytedance.com


Signed-off-by: default avatarQi Zheng <zhengqi.arch@bytedance.com>
Originally-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarKevin Brodsky <kevin.brodsky@arm.com>
Acked-by: Alexander Gordeev <agordeev@linux.ibm.com>	[s390]
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent b7dcd539

Documentation/mm/split_page_table_lock.rst

+2 −2
Original line number Diff line number Diff line
@@ -62,7 +62,7 @@ Support of split page table lock by an architecture 
===================================================



There's no need in special enabling of PTE split page table lock: everything

required is done by pagetable_pte_ctor() and pagetable_pte_dtor(), which

required is done by pagetable_pte_ctor() and pagetable_dtor(), which

must be called on PTE table allocation / freeing.



Make sure the architecture doesn't use slab allocator for page table
@@ -73,7 +73,7 @@ PMD split lock only makes sense if you have more than two page table 
levels.



PMD split lock enabling requires pagetable_pmd_ctor() call on PMD table

allocation and pagetable_pmd_dtor() on freeing.

allocation and pagetable_dtor() on freeing.



Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and

pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing

arch/arm/include/asm/tlb.h

+2 −2
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) 
{

	struct ptdesc *ptdesc = page_ptdesc(pte);



	pagetable_pte_dtor(ptdesc);

	pagetable_dtor(ptdesc);



#ifndef CONFIG_ARM_LPAE

	/*
@@ -61,7 +61,7 @@ __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) 
#ifdef CONFIG_ARM_LPAE

	struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);



	pagetable_pmd_dtor(ptdesc);

	pagetable_dtor(ptdesc);

	tlb_remove_ptdesc(tlb, ptdesc);

#endif

}

arch/arm64/include/asm/tlb.h

+4 −4
Original line number Diff line number Diff line
@@ -82,7 +82,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 
{

	struct ptdesc *ptdesc = page_ptdesc(pte);



	pagetable_pte_dtor(ptdesc);

	pagetable_dtor(ptdesc);

	tlb_remove_ptdesc(tlb, ptdesc);

}
@@ -92,7 +92,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 
{

	struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);



	pagetable_pmd_dtor(ptdesc);

	pagetable_dtor(ptdesc);

	tlb_remove_ptdesc(tlb, ptdesc);

}

#endif
@@ -106,7 +106,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, 
	if (!pgtable_l4_enabled())

		return;



	pagetable_pud_dtor(ptdesc);

	pagetable_dtor(ptdesc);

	tlb_remove_ptdesc(tlb, ptdesc);

}

#endif
@@ -120,7 +120,7 @@ static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp, 
	if (!pgtable_l5_enabled())

		return;



	pagetable_p4d_dtor(ptdesc);

	pagetable_dtor(ptdesc);

	tlb_remove_ptdesc(tlb, ptdesc);

}

#endif

arch/csky/include/asm/pgalloc.h

+1 −1
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) 


#define __pte_free_tlb(tlb, pte, address)		\

do {							\

	pagetable_pte_dtor(page_ptdesc(pte));		\

	pagetable_dtor(page_ptdesc(pte));		\

	tlb_remove_page_ptdesc(tlb, page_ptdesc(pte));	\

} while (0)

arch/hexagon/include/asm/pgalloc.h

+1 −1
Original line number Diff line number Diff line
@@ -89,7 +89,7 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, 


#define __pte_free_tlb(tlb, pte, addr)				\

do {								\

	pagetable_pte_dtor((page_ptdesc(pte)));			\

	pagetable_dtor((page_ptdesc(pte)));			\

	tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte)));	\

} while (0)