Unverified Commit e015eb62 authored by Palmer Dabbelt's avatar Palmer Dabbelt
Browse files

Merge patch series "riscv: Use READ_ONCE()/WRITE_ONCE() for pte accesses" 

Alexandre Ghiti <alexghiti@rivosinc.com> says:

This series is a follow-up for riscv of a recent series from Ryan [1] which
converts all direct dereferences of pte_t into a ptet_get() access.

The goal here for riscv is to use READ_ONCE()/WRITE_ONCE() for all page
table entries accesses to avoid any compiler transformation when the
hardware can concurrently modify the page tables entries (A/D bits for
example).

I went a bit further and added pud/p4d/pgd_get() helpers as such concurrent
modifications can happen too at those levels.

[1] https://lore.kernel.org/all/20230612151545.3317766-1-ryan.roberts@arm.com/

* b4-shazam-merge:
  riscv: Use accessors to page table entries instead of direct dereference
  riscv: mm: Only compile pgtable.c if MMU
  mm: Introduce pudp/p4dp/pgdp_get() functions
  riscv: Use WRITE_ONCE() when setting page table entries

Link: https://lore.kernel.org/r/20231213203001.179237-1-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parents 4a6b93f5 edf95564

arch/arm/include/asm/pgtable.h

+2 −0
Original line number Diff line number Diff line
@@ -151,6 +151,8 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 


extern pgd_t swapper_pg_dir[PTRS_PER_PGD];



#define pgdp_get(pgpd)		READ_ONCE(*pgdp)



#define pud_page(pud)		pmd_page(__pmd(pud_val(pud)))

#define pud_write(pud)		pmd_write(__pmd(pud_val(pud)))

arch/riscv/include/asm/kfence.h

+2 −2
Original line number Diff line number Diff line
@@ -18,9 +18,9 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) 
	pte_t *pte = virt_to_kpte(addr);



	if (protect)

		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));

		set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~_PAGE_PRESENT));

	else

		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));

		set_pte(pte, __pte(pte_val(ptep_get(pte)) | _PAGE_PRESENT));



	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);

arch/riscv/include/asm/pgtable-64.h

+5 −17
Original line number Diff line number Diff line
@@ -202,7 +202,7 @@ static inline int pud_user(pud_t pud) 


static inline void set_pud(pud_t *pudp, pud_t pud)

{

	*pudp = pud;

	WRITE_ONCE(*pudp, pud);

}



static inline void pud_clear(pud_t *pudp)
@@ -278,7 +278,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) 
static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)

{

	if (pgtable_l4_enabled)

		*p4dp = p4d;

		WRITE_ONCE(*p4dp, p4d);

	else

		set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) });

}
@@ -340,18 +340,12 @@ static inline struct page *p4d_page(p4d_t p4d) 
#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))



#define pud_offset pud_offset

static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)

{

	if (pgtable_l4_enabled)

		return p4d_pgtable(*p4d) + pud_index(address);



	return (pud_t *)p4d;

}

pud_t *pud_offset(p4d_t *p4d, unsigned long address);



static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)

{

	if (pgtable_l5_enabled)

		*pgdp = pgd;

		WRITE_ONCE(*pgdp, pgd);

	else

		set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) });

}
@@ -404,12 +398,6 @@ static inline struct page *pgd_page(pgd_t pgd) 
#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))



#define p4d_offset p4d_offset

static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)

{

	if (pgtable_l5_enabled)

		return pgd_pgtable(*pgd) + p4d_index(address);



	return (p4d_t *)pgd;

}

p4d_t *p4d_offset(pgd_t *pgd, unsigned long address);



#endif /* _ASM_RISCV_PGTABLE_64_H */

arch/riscv/include/asm/pgtable.h

+8 −25
Original line number Diff line number Diff line
@@ -248,7 +248,7 @@ static inline int pmd_leaf(pmd_t pmd) 


static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)

{

	*pmdp = pmd;

	WRITE_ONCE(*pmdp, pmd);

}



static inline void pmd_clear(pmd_t *pmdp)
@@ -510,7 +510,7 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) 
 */

static inline void set_pte(pte_t *ptep, pte_t pteval)

{

	*ptep = pteval;

	WRITE_ONCE(*ptep, pteval);

}



void flush_icache_pte(pte_t pte);
@@ -544,19 +544,12 @@ static inline void pte_clear(struct mm_struct *mm, 
	__set_pte_at(ptep, __pte(0));

}



#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS

static inline int ptep_set_access_flags(struct vm_area_struct *vma,

					unsigned long address, pte_t *ptep,

					pte_t entry, int dirty)

{

	if (!pte_same(*ptep, entry))

		__set_pte_at(ptep, entry);

	/*

	 * update_mmu_cache will unconditionally execute, handling both

	 * the case that the PTE changed and the spurious fault case.

	 */

	return true;

}

#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS	/* defined in mm/pgtable.c */

extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,

				 pte_t *ptep, pte_t entry, int dirty);

#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG	/* defined in mm/pgtable.c */

extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address,

				     pte_t *ptep);



#define __HAVE_ARCH_PTEP_GET_AND_CLEAR

static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
@@ -569,16 +562,6 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 
	return pte;

}



#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG

static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,

					    unsigned long address,

					    pte_t *ptep)

{

	if (!pte_young(*ptep))

		return 0;

	return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep));

}



#define __HAVE_ARCH_PTEP_SET_WRPROTECT

static inline void ptep_set_wrprotect(struct mm_struct *mm,

				      unsigned long address, pte_t *ptep)

arch/riscv/kernel/efi.c

+1 −1
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) 
static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data)

{

	efi_memory_desc_t *md = data;

	pte_t pte = READ_ONCE(*ptep);

	pte_t pte = ptep_get(ptep);

	unsigned long val;



	if (md->attribute & EFI_MEMORY_RO) {