Unverified Commit 648321fa authored by Jisheng Zhang's avatar Jisheng Zhang Committed by Palmer Dabbelt
Browse files

riscv: mm: try VMA lock-based page fault handling first 



Attempt VMA lock-based page fault handling first, and fall back to the
existing mmap_lock-based handling if that fails.

A simple running the ebizzy benchmark on Lichee Pi 4A shows that
PER_VMA_LOCK can improve the ebizzy benchmark by about 32.68%. In
theory, the more CPUs, the bigger improvement, but I don't have any
HW platform which has more than 4 CPUs.

This is the riscv variant of "x86/mm: try VMA lock-based page fault
handling first".

Signed-off-by: default avatarJisheng Zhang <jszhang@kernel.org>
Reviewed-by: default avatarGuo Ren <guoren@kernel.org>
Reviewed-by: default avatarSuren Baghdasaryan <surenb@google.com>
Link: https://lore.kernel.org/r/20230523165942.2630-1-jszhang@kernel.org


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 7d3332be

arch/riscv/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ config RISCV 
	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU

	select ARCH_SUPPORTS_HUGETLBFS if MMU

	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU

	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU

	select ARCH_USE_MEMTEST

	select ARCH_USE_QUEUED_RWLOCKS

	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU

arch/riscv/mm/fault.c

+33 −0
Original line number Diff line number Diff line
@@ -274,6 +274,36 @@ void handle_page_fault(struct pt_regs *regs) 
		flags |= FAULT_FLAG_WRITE;

	else if (cause == EXC_INST_PAGE_FAULT)

		flags |= FAULT_FLAG_INSTRUCTION;

#ifdef CONFIG_PER_VMA_LOCK

	if (!(flags & FAULT_FLAG_USER))

		goto lock_mmap;



	vma = lock_vma_under_rcu(mm, addr);

	if (!vma)

		goto lock_mmap;



	if (unlikely(access_error(cause, vma))) {

		vma_end_read(vma);

		goto lock_mmap;

	}



	fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs);

	vma_end_read(vma);



	if (!(fault & VM_FAULT_RETRY)) {

		count_vm_vma_lock_event(VMA_LOCK_SUCCESS);

		goto done;

	}

	count_vm_vma_lock_event(VMA_LOCK_RETRY);



	if (fault_signal_pending(fault, regs)) {

		if (!user_mode(regs))

			no_context(regs, addr);

		return;

	}

lock_mmap:

#endif /* CONFIG_PER_VMA_LOCK */



retry:

	mmap_read_lock(mm);

	vma = find_vma(mm, addr);
@@ -343,6 +373,9 @@ void handle_page_fault(struct pt_regs *regs) 


	mmap_read_unlock(mm);



#ifdef CONFIG_PER_VMA_LOCK

done:

#endif

	if (unlikely(fault & VM_FAULT_ERROR)) {

		tsk->thread.bad_cause = cause;

		mm_fault_error(regs, addr, fault);