git/linux-cryptodev-2.6
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git synced 2026-04-18 03:23:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

arm64/mm: Hoist barriers out of set_ptes_anysz() loop

Browse Source 
set_ptes_anysz() previously called __set_pte() for each PTE in the
range, which would conditionally issue a DSB and ISB to make the new PTE
value immediately visible to the table walker if the new PTE was valid
and for kernel space.

We can do better than this; let's hoist those barriers out of the loop
so that they are only issued once at the end of the loop. We then reduce
the cost by the number of PTEs in the range.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Tested-by: Luiz Capitulino <luizcap@redhat.com>
Link: https://lore.kernel.org/r/20250422081822.1836315-7-ryan.roberts@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Ryan Roberts
2025-04-22 09:18:14 +01:00
committed by Will Deacon
parent a899b7d067
commit f89b399e8d
1 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
arch/arm64/include/asm/pgtable.h
View File

@@ -320,13 +320,11 @@ static inline void __set_pte_nosync(pte_t *ptep, pte_t pte)
WRITE_ONCE(*ptep, pte);
}
static inline void __set_pte(pte_t *ptep, pte_t pte)
static inline void __set_pte_complete(pte_t pte)
{
__set_pte_nosync(ptep, pte);
/*
* Only if the new pte is valid and kernel, otherwise TLB maintenance
* or update_mmu_cache() have the necessary barriers.
* has the necessary barriers.
*/
if (pte_valid_not_user(pte)) {
dsb(ishst);
@@ -334,6 +332,12 @@ static inline void __set_pte(pte_t *ptep, pte_t pte)
}
}
static inline void __set_pte(pte_t *ptep, pte_t pte)
{
__set_pte_nosync(ptep, pte);
__set_pte_complete(pte);
}
static inline pte_t __ptep_get(pte_t *ptep)
{
return READ_ONCE(*ptep);
@@ -658,12 +662,14 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, pte_t *ptep,
for (;;) {
__check_safe_pte_update(mm, ptep, pte);
__set_pte(ptep, pte);
__set_pte_nosync(ptep, pte);
if (--nr == 0)
break;
ptep++;
pte = pte_advance_pfn(pte, stride);
}
__set_pte_complete(pte);
}
static inline void __set_ptes(struct mm_struct *mm,