Unverified Commit edf95564 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Use accessors to page table entries instead of direct dereference 



As very well explained in commit 20a004e7 ("arm64: mm: Use
READ_ONCE/WRITE_ONCE when accessing page tables"), an architecture whose
page table walker can modify the PTE in parallel must use
READ_ONCE()/WRITE_ONCE() macro to avoid any compiler transformation.

So apply that to riscv which is such architecture.

Signed-off-by: default avatarAlexandre Ghiti <alexghiti@rivosinc.com>
Acked-by: default avatarAnup Patel <anup@brainfault.org>
Link: https://lore.kernel.org/r/20231213203001.179237-5-alexghiti@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent d6508999

arch/riscv/include/asm/kfence.h

+2 −2
Original line number Diff line number Diff line
@@ -18,9 +18,9 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) 
	pte_t *pte = virt_to_kpte(addr);



	if (protect)

		set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));

		set_pte(pte, __pte(pte_val(ptep_get(pte)) & ~_PAGE_PRESENT));

	else

		set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT));

		set_pte(pte, __pte(pte_val(ptep_get(pte)) | _PAGE_PRESENT));



	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);

arch/riscv/include/asm/pgtable-64.h

+2 −14
Original line number Diff line number Diff line
@@ -340,13 +340,7 @@ static inline struct page *p4d_page(p4d_t p4d) 
#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))



#define pud_offset pud_offset

static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)

{

	if (pgtable_l4_enabled)

		return p4d_pgtable(*p4d) + pud_index(address);



	return (pud_t *)p4d;

}

pud_t *pud_offset(p4d_t *p4d, unsigned long address);



static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)

{
@@ -404,12 +398,6 @@ static inline struct page *pgd_page(pgd_t pgd) 
#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))



#define p4d_offset p4d_offset

static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)

{

	if (pgtable_l5_enabled)

		return pgd_pgtable(*pgd) + p4d_index(address);



	return (p4d_t *)pgd;

}

p4d_t *p4d_offset(pgd_t *pgd, unsigned long address);



#endif /* _ASM_RISCV_PGTABLE_64_H */

arch/riscv/include/asm/pgtable.h

+6 −23
Original line number Diff line number Diff line
@@ -544,19 +544,12 @@ static inline void pte_clear(struct mm_struct *mm, 
	__set_pte_at(ptep, __pte(0));

}



#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS

static inline int ptep_set_access_flags(struct vm_area_struct *vma,

					unsigned long address, pte_t *ptep,

					pte_t entry, int dirty)

{

	if (!pte_same(*ptep, entry))

		__set_pte_at(ptep, entry);

	/*

	 * update_mmu_cache will unconditionally execute, handling both

	 * the case that the PTE changed and the spurious fault case.

	 */

	return true;

}

#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS	/* defined in mm/pgtable.c */

extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,

				 pte_t *ptep, pte_t entry, int dirty);

#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG	/* defined in mm/pgtable.c */

extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address,

				     pte_t *ptep);



#define __HAVE_ARCH_PTEP_GET_AND_CLEAR

static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
@@ -569,16 +562,6 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 
	return pte;

}



#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG

static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,

					    unsigned long address,

					    pte_t *ptep)

{

	if (!pte_young(*ptep))

		return 0;

	return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep));

}



#define __HAVE_ARCH_PTEP_SET_WRPROTECT

static inline void ptep_set_wrprotect(struct mm_struct *mm,

				      unsigned long address, pte_t *ptep)

arch/riscv/kernel/efi.c

+1 −1
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) 
static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data)

{

	efi_memory_desc_t *md = data;

	pte_t pte = READ_ONCE(*ptep);

	pte_t pte = ptep_get(ptep);

	unsigned long val;



	if (md->attribute & EFI_MEMORY_RO) {

arch/riscv/kvm/mmu.c

+11 −11
Original line number Diff line number Diff line
@@ -103,7 +103,7 @@ static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, 
	*ptep_level = current_level;

	ptep = (pte_t *)kvm->arch.pgd;

	ptep = &ptep[gstage_pte_index(addr, current_level)];

	while (ptep && pte_val(*ptep)) {

	while (ptep && pte_val(ptep_get(ptep))) {

		if (gstage_pte_leaf(ptep)) {

			*ptep_level = current_level;

			*ptepp = ptep;
@@ -113,7 +113,7 @@ static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, 
		if (current_level) {

			current_level--;

			*ptep_level = current_level;

			ptep = (pte_t *)gstage_pte_page_vaddr(*ptep);

			ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));

			ptep = &ptep[gstage_pte_index(addr, current_level)];

		} else {

			ptep = NULL;
@@ -149,25 +149,25 @@ static int gstage_set_pte(struct kvm *kvm, u32 level, 
		if (gstage_pte_leaf(ptep))

			return -EEXIST;



		if (!pte_val(*ptep)) {

		if (!pte_val(ptep_get(ptep))) {

			if (!pcache)

				return -ENOMEM;

			next_ptep = kvm_mmu_memory_cache_alloc(pcache);

			if (!next_ptep)

				return -ENOMEM;

			*ptep = pfn_pte(PFN_DOWN(__pa(next_ptep)),

					__pgprot(_PAGE_TABLE));

			set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)),

					      __pgprot(_PAGE_TABLE)));

		} else {

			if (gstage_pte_leaf(ptep))

				return -EEXIST;

			next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep);

			next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));

		}



		current_level--;

		ptep = &next_ptep[gstage_pte_index(addr, current_level)];

	}



	*ptep = *new_pte;

	set_pte(ptep, *new_pte);

	if (gstage_pte_leaf(ptep))

		gstage_remote_tlb_flush(kvm, current_level, addr);
@@ -239,11 +239,11 @@ static void gstage_op_pte(struct kvm *kvm, gpa_t addr, 


	BUG_ON(addr & (page_size - 1));



	if (!pte_val(*ptep))

	if (!pte_val(ptep_get(ptep)))

		return;



	if (ptep_level && !gstage_pte_leaf(ptep)) {

		next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep);

		next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep));

		next_ptep_level = ptep_level - 1;

		ret = gstage_level_to_page_size(next_ptep_level,

						&next_page_size);
@@ -261,7 +261,7 @@ static void gstage_op_pte(struct kvm *kvm, gpa_t addr, 
		if (op == GSTAGE_OP_CLEAR)

			set_pte(ptep, __pte(0));

		else if (op == GSTAGE_OP_WP)

			set_pte(ptep, __pte(pte_val(*ptep) & ~_PAGE_WRITE));

			set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE));

		gstage_remote_tlb_flush(kvm, ptep_level, addr);

	}

}
@@ -603,7 +603,7 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) 
				   &ptep, &ptep_level))

		return false;



	return pte_young(*ptep);

	return pte_young(ptep_get(ptep));

}



int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,