Commit ce6b5ff5 authored by Kristina Martsenko's avatar Kristina Martsenko Committed by Catalin Marinas
Browse files

arm64: lib: Use MOPS for copy_page() and clear_page() 



Similarly to what was done to the memcpy() routines, make copy_page()
and clear_page() also use the Armv8.8 FEAT_MOPS instructions.

Note: For copy_page() this uses the CPY* instructions instead of CPYF*
as CPYF* doesn't allow src and dst to be equal. It's not clear if
copy_page() needs to allow equal src and dst but it has worked so far
with the current implementation and there is no documentation forbidding
it.

Note, the unoptimized version of copy_page() in assembler.h is left as
it is.

Signed-off-by: default avatarKristina Martsenko <kristina.martsenko@arm.com>
Link: https://lore.kernel.org/r/20240930161051.3777828-6-kristina.martsenko@arm.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 836ed3c4

arch/arm64/lib/clear_page.S

+13 −0
Original line number Diff line number Diff line
@@ -15,6 +15,19 @@ 
 *	x0 - dest

 */

SYM_FUNC_START(__pi_clear_page)

#ifdef CONFIG_AS_HAS_MOPS

	.arch_extension mops

alternative_if_not ARM64_HAS_MOPS

	b	.Lno_mops

alternative_else_nop_endif



	mov	x1, #PAGE_SIZE

	setpn	[x0]!, x1!, xzr

	setmn	[x0]!, x1!, xzr

	seten	[x0]!, x1!, xzr

	ret

.Lno_mops:

#endif

	mrs	x1, dczid_el0

	tbnz	x1, #4, 2f	/* Branch if DC ZVA is prohibited */

	and	w1, w1, #0xf

arch/arm64/lib/copy_page.S

+13 −0
Original line number Diff line number Diff line
@@ -18,6 +18,19 @@ 
 *	x1 - src

 */

SYM_FUNC_START(__pi_copy_page)

#ifdef CONFIG_AS_HAS_MOPS

	.arch_extension mops

alternative_if_not ARM64_HAS_MOPS

	b	.Lno_mops

alternative_else_nop_endif



	mov	x2, #PAGE_SIZE

	cpypwn	[x0]!, [x1]!, x2!

	cpymwn	[x0]!, [x1]!, x2!

	cpyewn	[x0]!, [x1]!, x2!

	ret

.Lno_mops:

#endif

	ldp	x2, x3, [x1]

	ldp	x4, x5, [x1, #16]

	ldp	x6, x7, [x1, #32]