Commit c7349719 authored by Will Deacon's avatar Will Deacon
Browse files

Merge branch 'for-next/mm' into for-next/core 

* for-next/mm:
  arm64/boot: Disallow BSS exports to startup code
  arm64/boot: Move global CPU override variables out of BSS
  arm64/boot: Move init_pgdir[] and init_idmap_pgdir[] into __pi_ namespace
  arm64: mm: Drop redundant check in pmd_trans_huge()
  arm64/mm: Permit lazy_mmu_mode to be nested
  arm64/mm: Disable barrier batching in interrupt contexts
  arm64/mm: Batch barriers when updating kernel mappings
  mm/vmalloc: Enter lazy mmu mode while manipulating vmalloc ptes
  arm64/mm: Support huge pte-mapped pages in vmap
  mm/vmalloc: Gracefully unmap huge ptes
  mm/vmalloc: Warn on improper use of vunmap_range()
  arm64/mm: Hoist barriers out of set_ptes_anysz() loop
  arm64: hugetlb: Use __set_ptes_anysz() and __ptep_get_and_clear_anysz()
  arm64/mm: Refactor __set_ptes() and __ptep_get_and_clear()
  mm/page_table_check: Batch-check pmds/puds just like ptes
  arm64: hugetlb: Refine tlb maintenance scope
  arm64: hugetlb: Cleanup huge_pte size discovery mechanisms
  arm64: pageattr: Explicitly bail out when changing permissions for vmalloc_huge mappings
  arm64: Support ARM64_VA_BITS=52 when setting ARCH_MMAP_RND_BITS_MAX
  arm64/mm: Remove randomization of the linear map
parents 9d27622f 90530521

arch/arm64/Kconfig

+3 −3
Original line number Diff line number Diff line
@@ -333,9 +333,9 @@ config ARCH_MMAP_RND_BITS_MAX 
	default 24 if ARM64_VA_BITS=39

	default 27 if ARM64_VA_BITS=42

	default 30 if ARM64_VA_BITS=47

	default 29 if ARM64_VA_BITS=48 && ARM64_64K_PAGES

	default 31 if ARM64_VA_BITS=48 && ARM64_16K_PAGES

	default 33 if ARM64_VA_BITS=48

	default 29 if (ARM64_VA_BITS=48 || ARM64_VA_BITS=52) && ARM64_64K_PAGES

	default 31 if (ARM64_VA_BITS=48 || ARM64_VA_BITS=52) && ARM64_16K_PAGES

	default 33 if (ARM64_VA_BITS=48 || ARM64_VA_BITS=52)

	default 14 if ARM64_64K_PAGES

	default 16 if ARM64_16K_PAGES

	default 18

arch/arm64/include/asm/hugetlb.h

+19 −10
Original line number Diff line number Diff line
@@ -69,29 +69,38 @@ extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, 


#include <asm-generic/hugetlb.h>



#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE

static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,

static inline void __flush_hugetlb_tlb_range(struct vm_area_struct *vma,

					     unsigned long start,

					   unsigned long end)

					     unsigned long end,

					     unsigned long stride,

					     bool last_level)

{

	unsigned long stride = huge_page_size(hstate_vma(vma));



	switch (stride) {

#ifndef __PAGETABLE_PMD_FOLDED

	case PUD_SIZE:

		__flush_tlb_range(vma, start, end, PUD_SIZE, false, 1);

		__flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);

		break;

#endif

	case CONT_PMD_SIZE:

	case PMD_SIZE:

		__flush_tlb_range(vma, start, end, PMD_SIZE, false, 2);

		__flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);

		break;

	case CONT_PTE_SIZE:

		__flush_tlb_range(vma, start, end, PAGE_SIZE, false, 3);

		__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);

		break;

	default:

		__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);

		__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);

	}

}



#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE

static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,

					   unsigned long start,

					   unsigned long end)

{

	unsigned long stride = huge_page_size(hstate_vma(vma));



	__flush_hugetlb_tlb_range(vma, start, end, stride, false);

}



#endif /* __ASM_HUGETLB_H */

arch/arm64/include/asm/pgtable.h

+171 −65
Original line number Diff line number Diff line
@@ -40,6 +40,85 @@ 
#include <linux/sched.h>

#include <linux/page_table_check.h>



static inline void emit_pte_barriers(void)

{

	/*

	 * These barriers are emitted under certain conditions after a pte entry

	 * was modified (see e.g. __set_pte_complete()). The dsb makes the store

	 * visible to the table walker. The isb ensures that any previous

	 * speculative "invalid translation" marker that is in the CPU's

	 * pipeline gets cleared, so that any access to that address after

	 * setting the pte to valid won't cause a spurious fault. If the thread

	 * gets preempted after storing to the pgtable but before emitting these

	 * barriers, __switch_to() emits a dsb which ensure the walker gets to

	 * see the store. There is no guarantee of an isb being issued though.

	 * This is safe because it will still get issued (albeit on a

	 * potentially different CPU) when the thread starts running again,

	 * before any access to the address.

	 */

	dsb(ishst);

	isb();

}



static inline void queue_pte_barriers(void)

{

	unsigned long flags;



	if (in_interrupt()) {

		emit_pte_barriers();

		return;

	}



	flags = read_thread_flags();



	if (flags & BIT(TIF_LAZY_MMU)) {

		/* Avoid the atomic op if already set. */

		if (!(flags & BIT(TIF_LAZY_MMU_PENDING)))

			set_thread_flag(TIF_LAZY_MMU_PENDING);

	} else {

		emit_pte_barriers();

	}

}



#define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE

static inline void arch_enter_lazy_mmu_mode(void)

{

	/*

	 * lazy_mmu_mode is not supposed to permit nesting. But in practice this

	 * does happen with CONFIG_DEBUG_PAGEALLOC, where a page allocation

	 * inside a lazy_mmu_mode section (such as zap_pte_range()) will change

	 * permissions on the linear map with apply_to_page_range(), which

	 * re-enters lazy_mmu_mode. So we tolerate nesting in our

	 * implementation. The first call to arch_leave_lazy_mmu_mode() will

	 * flush and clear the flag such that the remainder of the work in the

	 * outer nest behaves as if outside of lazy mmu mode. This is safe and

	 * keeps tracking simple.

	 */



	if (in_interrupt())

		return;



	set_thread_flag(TIF_LAZY_MMU);

}



static inline void arch_flush_lazy_mmu_mode(void)

{

	if (in_interrupt())

		return;



	if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING))

		emit_pte_barriers();

}



static inline void arch_leave_lazy_mmu_mode(void)

{

	if (in_interrupt())

		return;



	arch_flush_lazy_mmu_mode();

	clear_thread_flag(TIF_LAZY_MMU);

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
@@ -320,18 +399,20 @@ static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) 
	WRITE_ONCE(*ptep, pte);

}



static inline void __set_pte(pte_t *ptep, pte_t pte)

static inline void __set_pte_complete(pte_t pte)

{

	__set_pte_nosync(ptep, pte);



	/*

	 * Only if the new pte is valid and kernel, otherwise TLB maintenance

	 * or update_mmu_cache() have the necessary barriers.

	 * has the necessary barriers.

	 */

	if (pte_valid_not_user(pte)) {

		dsb(ishst);

		isb();

	if (pte_valid_not_user(pte))

		queue_pte_barriers();

}



static inline void __set_pte(pte_t *ptep, pte_t pte)

{

	__set_pte_nosync(ptep, pte);

	__set_pte_complete(pte);

}



static inline pte_t __ptep_get(pte_t *ptep)
@@ -423,23 +504,6 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) 
	return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));

}



static inline void __set_ptes(struct mm_struct *mm,

			      unsigned long __always_unused addr,

			      pte_t *ptep, pte_t pte, unsigned int nr)

{

	page_table_check_ptes_set(mm, ptep, pte, nr);

	__sync_cache_and_tags(pte, nr);



	for (;;) {

		__check_safe_pte_update(mm, ptep, pte);

		__set_pte(ptep, pte);

		if (--nr == 0)

			break;

		ptep++;

		pte = pte_advance_pfn(pte, 1);

	}

}



/*

 * Hugetlb definitions.

 */
@@ -649,30 +713,64 @@ static inline pgprot_t pud_pgprot(pud_t pud) 
	return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud));

}



static inline void __set_pte_at(struct mm_struct *mm,

static inline void __set_ptes_anysz(struct mm_struct *mm, pte_t *ptep,

				    pte_t pte, unsigned int nr,

				    unsigned long pgsize)

{

	unsigned long stride = pgsize >> PAGE_SHIFT;



	switch (pgsize) {

	case PAGE_SIZE:

		page_table_check_ptes_set(mm, ptep, pte, nr);

		break;

	case PMD_SIZE:

		page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);

		break;

#ifndef __PAGETABLE_PMD_FOLDED

	case PUD_SIZE:

		page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);

		break;

#endif

	default:

		VM_WARN_ON(1);

	}



	__sync_cache_and_tags(pte, nr * stride);



	for (;;) {

		__check_safe_pte_update(mm, ptep, pte);

		__set_pte_nosync(ptep, pte);

		if (--nr == 0)

			break;

		ptep++;

		pte = pte_advance_pfn(pte, stride);

	}



	__set_pte_complete(pte);

}



static inline void __set_ptes(struct mm_struct *mm,

			      unsigned long __always_unused addr,

			      pte_t *ptep, pte_t pte, unsigned int nr)

{

	__sync_cache_and_tags(pte, nr);

	__check_safe_pte_update(mm, ptep, pte);

	__set_pte(ptep, pte);

	__set_ptes_anysz(mm, ptep, pte, nr, PAGE_SIZE);

}



static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,

			      pmd_t *pmdp, pmd_t pmd)

static inline void __set_pmds(struct mm_struct *mm,

			      unsigned long __always_unused addr,

			      pmd_t *pmdp, pmd_t pmd, unsigned int nr)

{

	page_table_check_pmd_set(mm, pmdp, pmd);

	return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd),

						PMD_SIZE >> PAGE_SHIFT);

	__set_ptes_anysz(mm, (pte_t *)pmdp, pmd_pte(pmd), nr, PMD_SIZE);

}

#define set_pmd_at(mm, addr, pmdp, pmd) __set_pmds(mm, addr, pmdp, pmd, 1)



static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,

			      pud_t *pudp, pud_t pud)

static inline void __set_puds(struct mm_struct *mm,

			      unsigned long __always_unused addr,

			      pud_t *pudp, pud_t pud, unsigned int nr)

{

	page_table_check_pud_set(mm, pudp, pud);

	return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud),

						PUD_SIZE >> PAGE_SHIFT);

	__set_ptes_anysz(mm, (pte_t *)pudp, pud_pte(pud), nr, PUD_SIZE);

}

#define set_pud_at(mm, addr, pudp, pud) __set_puds(mm, addr, pudp, pud, 1)



#define __p4d_to_phys(p4d)	__pte_to_phys(p4d_pte(p4d))

#define __phys_to_p4d_val(phys)	__phys_to_pte_val(phys)
@@ -739,8 +837,7 @@ static inline int pmd_trans_huge(pmd_t pmd) 
	 * If pmd is present-invalid, pmd_table() won't detect it

	 * as a table, so force the valid bit for the comparison.

	 */

	return pmd_val(pmd) && pmd_present(pmd) &&

	       !pmd_table(__pmd(pmd_val(pmd) | PTE_VALID));

	return pmd_present(pmd) && !pmd_table(__pmd(pmd_val(pmd) | PTE_VALID));

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
@@ -754,8 +851,6 @@ static inline bool pud_table(pud_t pud) { return true; } 
				 PUD_TYPE_TABLE)

#endif



extern pgd_t init_pg_dir[];

extern pgd_t init_pg_end[];

extern pgd_t swapper_pg_dir[];

extern pgd_t idmap_pg_dir[];

extern pgd_t tramp_pg_dir[];
@@ -780,10 +875,8 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 


	WRITE_ONCE(*pmdp, pmd);



	if (pmd_valid(pmd)) {

		dsb(ishst);

		isb();

	}

	if (pmd_valid(pmd))

		queue_pte_barriers();

}



static inline void pmd_clear(pmd_t *pmdp)
@@ -848,10 +941,8 @@ static inline void set_pud(pud_t *pudp, pud_t pud) 


	WRITE_ONCE(*pudp, pud);



	if (pud_valid(pud)) {

		dsb(ishst);

		isb();

	}

	if (pud_valid(pud))

		queue_pte_barriers();

}



static inline void pud_clear(pud_t *pudp)
@@ -930,8 +1021,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 
	}



	WRITE_ONCE(*p4dp, p4d);

	dsb(ishst);

	isb();

	queue_pte_barriers();

}



static inline void p4d_clear(p4d_t *p4dp)
@@ -1059,8 +1149,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 
	}



	WRITE_ONCE(*pgdp, pgd);

	dsb(ishst);

	isb();

	queue_pte_barriers();

}



static inline void pgd_clear(pgd_t *pgdp)
@@ -1301,16 +1390,37 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 
}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */



static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,

				       unsigned long address, pte_t *ptep)

static inline pte_t __ptep_get_and_clear_anysz(struct mm_struct *mm,

					       pte_t *ptep,

					       unsigned long pgsize)

{

	pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));



	switch (pgsize) {

	case PAGE_SIZE:

		page_table_check_pte_clear(mm, pte);

		break;

	case PMD_SIZE:

		page_table_check_pmd_clear(mm, pte_pmd(pte));

		break;

#ifndef __PAGETABLE_PMD_FOLDED

	case PUD_SIZE:

		page_table_check_pud_clear(mm, pte_pud(pte));

		break;

#endif

	default:

		VM_WARN_ON(1);

	}



	return pte;

}



static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,

				       unsigned long address, pte_t *ptep)

{

	return __ptep_get_and_clear_anysz(mm, ptep, PAGE_SIZE);

}



static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,

				pte_t *ptep, unsigned int nr, int full)

{
@@ -1347,11 +1457,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, 
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,

					    unsigned long address, pmd_t *pmdp)

{

	pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0));



	page_table_check_pmd_clear(mm, pmd);



	return pmd;

	return pte_pmd(__ptep_get_and_clear_anysz(mm, (pte_t *)pmdp, PMD_SIZE));

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

arch/arm64/include/asm/thread_info.h

+2 −0
Original line number Diff line number Diff line
@@ -83,6 +83,8 @@ void arch_setup_new_exec(void); 
#define TIF_SME_VL_INHERIT	28	/* Inherit SME vl_onexec across exec */

#define TIF_KERNEL_FPSTATE	29	/* Task is in a kernel mode FPSIMD section */

#define TIF_TSC_SIGSEGV		30	/* SIGSEGV on counter-timer access */

#define TIF_LAZY_MMU		31	/* Task in lazy mmu mode */

#define TIF_LAZY_MMU_PENDING	32	/* Ops pending for lazy mmu mode exit */



#define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)

#define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)

arch/arm64/include/asm/vmalloc.h

+45 −0
Original line number Diff line number Diff line
@@ -23,6 +23,51 @@ static inline bool arch_vmap_pmd_supported(pgprot_t prot) 
	return !IS_ENABLED(CONFIG_PTDUMP_DEBUGFS);

}



#define arch_vmap_pte_range_map_size arch_vmap_pte_range_map_size

static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr,

						unsigned long end, u64 pfn,

						unsigned int max_page_shift)

{

	/*

	 * If the block is at least CONT_PTE_SIZE in size, and is naturally

	 * aligned in both virtual and physical space, then we can pte-map the

	 * block using the PTE_CONT bit for more efficient use of the TLB.

	 */

	if (max_page_shift < CONT_PTE_SHIFT)

		return PAGE_SIZE;



	if (end - addr < CONT_PTE_SIZE)

		return PAGE_SIZE;



	if (!IS_ALIGNED(addr, CONT_PTE_SIZE))

		return PAGE_SIZE;



	if (!IS_ALIGNED(PFN_PHYS(pfn), CONT_PTE_SIZE))

		return PAGE_SIZE;



	return CONT_PTE_SIZE;

}



#define arch_vmap_pte_range_unmap_size arch_vmap_pte_range_unmap_size

static inline unsigned long arch_vmap_pte_range_unmap_size(unsigned long addr,

							   pte_t *ptep)

{

	/*

	 * The caller handles alignment so it's sufficient just to check

	 * PTE_CONT.

	 */

	return pte_valid_cont(__ptep_get(ptep)) ? CONT_PTE_SIZE : PAGE_SIZE;

}



#define arch_vmap_pte_supported_shift arch_vmap_pte_supported_shift

static inline int arch_vmap_pte_supported_shift(unsigned long size)

{

	if (size >= CONT_PTE_SIZE)

		return CONT_PTE_SHIFT;



	return PAGE_SHIFT;

}



#endif



#define arch_vmap_pgprot_tagged arch_vmap_pgprot_tagged