Commit 0cec9541 authored by Kefeng Wang's avatar Kefeng Wang Committed by Andrew Morton
Browse files

powerpc: mm: accelerate pagefault when badaccess 

The access_[pkey]_error() of vma already checked under per-VMA lock, if it
is a bad access, directly handle error, no need to retry with mmap_lock
again.  In order to release the correct lock, pass the mm_struct into
bad_access_pkey()/bad_access(), if mm is NULL, release vma lock, or
release mmap_lock.  Since the page faut is handled under per-VMA lock,
count it as a vma lock event with VMA_LOCK_SUCCESS.

Link: https://lkml.kernel.org/r/20240403083805.1818160-5-wangkefeng.wang@huawei.com


Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 15e4a5f5

arch/powerpc/mm/fault.c

+20 −13
Original line number Diff line number Diff line
@@ -71,23 +71,26 @@ static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long add 
	return __bad_area_nosemaphore(regs, address, SEGV_MAPERR);

}



static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code)

static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code,

		      struct mm_struct *mm, struct vm_area_struct *vma)

{

	struct mm_struct *mm = current->mm;



	/*

	 * Something tried to access memory that isn't in our memory map..

	 * Fix it, but check if it's kernel or user first..

	 */

	if (mm)

		mmap_read_unlock(mm);

	else

		vma_end_read(vma);



	return __bad_area_nosemaphore(regs, address, si_code);

}



static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address,

				    struct mm_struct *mm,

				    struct vm_area_struct *vma)

{

	struct mm_struct *mm = current->mm;

	int pkey;



	/*
@@ -109,7 +112,10 @@ static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, 
	 */

	pkey = vma_pkey(vma);



	if (mm)

		mmap_read_unlock(mm);

	else

		vma_end_read(vma);



	/*

	 * If we are in kernel mode, bail out with a SEGV, this will
@@ -124,9 +130,10 @@ static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, 
	return 0;

}



static noinline int bad_access(struct pt_regs *regs, unsigned long address)

static noinline int bad_access(struct pt_regs *regs, unsigned long address,

			       struct mm_struct *mm, struct vm_area_struct *vma)

{

	return __bad_area(regs, address, SEGV_ACCERR);

	return __bad_area(regs, address, SEGV_ACCERR, mm, vma);

}



static int do_sigbus(struct pt_regs *regs, unsigned long address,
@@ -479,13 +486,13 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, 


	if (unlikely(access_pkey_error(is_write, is_exec,

				       (error_code & DSISR_KEYFAULT), vma))) {

		vma_end_read(vma);

		goto lock_mmap;

		count_vm_vma_lock_event(VMA_LOCK_SUCCESS);

		return bad_access_pkey(regs, address, NULL, vma);

	}



	if (unlikely(access_error(is_write, is_exec, vma))) {

		vma_end_read(vma);

		goto lock_mmap;

		count_vm_vma_lock_event(VMA_LOCK_SUCCESS);

		return bad_access(regs, address, NULL, vma);

	}



	fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
@@ -521,10 +528,10 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, 


	if (unlikely(access_pkey_error(is_write, is_exec,

				       (error_code & DSISR_KEYFAULT), vma)))

		return bad_access_pkey(regs, address, vma);

		return bad_access_pkey(regs, address, mm, vma);



	if (unlikely(access_error(is_write, is_exec, vma)))

		return bad_access(regs, address);

		return bad_access(regs, address, mm, vma);



	/*

	 * If for any reason at all we couldn't handle the fault,