Commit 297565aa authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu
Browse files

lib/xor: make xor prototypes more friendly to compiler vectorization 



Modern compilers are perfectly capable of extracting parallelism from
the XOR routines, provided that the prototypes reflect the nature of the
input accurately, in particular, the fact that the input vectors are
expected not to overlap. This is not documented explicitly, but is
implied by the interchangeability of the various C routines, some of
which use temporary variables while others don't: this means that these
routines only behave identically for non-overlapping inputs.

So let's decorate these input vectors with the __restrict modifier,
which informs the compiler that there is no overlap. While at it, make
the input-only vectors pointer-to-const as well.

Tested-by: default avatarNathan Chancellor <nathan@kernel.org>
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Reviewed-by: default avatarNick Desaulniers <ndesaulniers@google.com>
Link: https://github.com/ClangBuiltLinux/linux/issues/563


Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent e8bf24bd

arch/alpha/include/asm/xor.h

+36 −17
Original line number Diff line number Diff line
@@ -5,24 +5,43 @@ 
 * Optimized RAID-5 checksumming functions for alpha EV5 and EV6

 */



extern void xor_alpha_2(unsigned long, unsigned long *, unsigned long *);

extern void xor_alpha_3(unsigned long, unsigned long *, unsigned long *,

		        unsigned long *);

extern void xor_alpha_4(unsigned long, unsigned long *, unsigned long *,

		        unsigned long *, unsigned long *);

extern void xor_alpha_5(unsigned long, unsigned long *, unsigned long *,

		        unsigned long *, unsigned long *, unsigned long *);

extern void

xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,

	    const unsigned long * __restrict p2);

extern void

xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,

	    const unsigned long * __restrict p2,

	    const unsigned long * __restrict p3);

extern void

xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,

	    const unsigned long * __restrict p2,

	    const unsigned long * __restrict p3,

	    const unsigned long * __restrict p4);

extern void

xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,

	    const unsigned long * __restrict p2,

	    const unsigned long * __restrict p3,

	    const unsigned long * __restrict p4,

	    const unsigned long * __restrict p5);



extern void xor_alpha_prefetch_2(unsigned long, unsigned long *,

				 unsigned long *);

extern void xor_alpha_prefetch_3(unsigned long, unsigned long *,

				 unsigned long *, unsigned long *);

extern void xor_alpha_prefetch_4(unsigned long, unsigned long *,

				 unsigned long *, unsigned long *,

				 unsigned long *);

extern void xor_alpha_prefetch_5(unsigned long, unsigned long *,

				 unsigned long *, unsigned long *,

				 unsigned long *, unsigned long *);

extern void

xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,

		     const unsigned long * __restrict p2);

extern void

xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,

		     const unsigned long * __restrict p2,

		     const unsigned long * __restrict p3);

extern void

xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,

		     const unsigned long * __restrict p2,

		     const unsigned long * __restrict p3,

		     const unsigned long * __restrict p4);

extern void

xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,

		     const unsigned long * __restrict p2,

		     const unsigned long * __restrict p3,

		     const unsigned long * __restrict p4,

		     const unsigned long * __restrict p5);



asm("								\n\

	.text							\n\

arch/arm/include/asm/xor.h

+28 −14
Original line number Diff line number Diff line
@@ -44,7 +44,8 @@ 
		: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))



static void

xor_arm4regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)

xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,

	       const unsigned long * __restrict p2)

{

	unsigned int lines = bytes / sizeof(unsigned long) / 4;

	register unsigned int a1 __asm__("r4");
@@ -64,8 +65,9 @@ xor_arm4regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 
}



static void

xor_arm4regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3)

xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,

	       const unsigned long * __restrict p2,

	       const unsigned long * __restrict p3)

{

	unsigned int lines = bytes / sizeof(unsigned long) / 4;

	register unsigned int a1 __asm__("r4");
@@ -86,8 +88,10 @@ xor_arm4regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, 
}



static void

xor_arm4regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3, unsigned long *p4)

xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,

	       const unsigned long * __restrict p2,

	       const unsigned long * __restrict p3,

	       const unsigned long * __restrict p4)

{

	unsigned int lines = bytes / sizeof(unsigned long) / 2;

	register unsigned int a1 __asm__("r8");
@@ -105,8 +109,11 @@ xor_arm4regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, 
}



static void

xor_arm4regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3, unsigned long *p4, unsigned long *p5)

xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,

	       const unsigned long * __restrict p2,

	       const unsigned long * __restrict p3,

	       const unsigned long * __restrict p4,

	       const unsigned long * __restrict p5)

{

	unsigned int lines = bytes / sizeof(unsigned long) / 2;

	register unsigned int a1 __asm__("r8");
@@ -146,7 +153,8 @@ static struct xor_block_template xor_block_arm4regs = { 
extern struct xor_block_template const xor_block_neon_inner;



static void

xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)

xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2)

{

	if (in_interrupt()) {

		xor_arm4regs_2(bytes, p1, p2);
@@ -158,8 +166,9 @@ xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 
}



static void

xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3)

xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2,

	   const unsigned long * __restrict p3)

{

	if (in_interrupt()) {

		xor_arm4regs_3(bytes, p1, p2, p3);
@@ -171,8 +180,10 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, 
}



static void

xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3, unsigned long *p4)

xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2,

	   const unsigned long * __restrict p3,

	   const unsigned long * __restrict p4)

{

	if (in_interrupt()) {

		xor_arm4regs_4(bytes, p1, p2, p3, p4);
@@ -184,8 +195,11 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, 
}



static void

xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3, unsigned long *p4, unsigned long *p5)

xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2,

	   const unsigned long * __restrict p3,

	   const unsigned long * __restrict p4,

	   const unsigned long * __restrict p5)

{

	if (in_interrupt()) {

		xor_arm4regs_5(bytes, p1, p2, p3, p4, p5);

arch/arm64/include/asm/xor.h

+14 −7
Original line number Diff line number Diff line
@@ -16,7 +16,8 @@ 
extern struct xor_block_template const xor_block_inner_neon;



static void

xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)

xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2)

{

	kernel_neon_begin();

	xor_block_inner_neon.do_2(bytes, p1, p2);
@@ -24,8 +25,9 @@ xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 
}



static void

xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3)

xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2,

	   const unsigned long * __restrict p3)

{

	kernel_neon_begin();

	xor_block_inner_neon.do_3(bytes, p1, p2, p3);
@@ -33,8 +35,10 @@ xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, 
}



static void

xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3, unsigned long *p4)

xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2,

	   const unsigned long * __restrict p3,

	   const unsigned long * __restrict p4)

{

	kernel_neon_begin();

	xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
@@ -42,8 +46,11 @@ xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, 
}



static void

xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,

		unsigned long *p3, unsigned long *p4, unsigned long *p5)

xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,

	   const unsigned long * __restrict p2,

	   const unsigned long * __restrict p3,

	   const unsigned long * __restrict p4,

	   const unsigned long * __restrict p5)

{

	kernel_neon_begin();

	xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);

arch/arm64/lib/xor-neon.c

+29 −17
Original line number Diff line number Diff line
@@ -10,8 +10,8 @@ 
#include <linux/module.h>

#include <asm/neon-intrinsics.h>



void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1,

	unsigned long *p2)

void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,

	const unsigned long * __restrict p2)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;
@@ -37,8 +37,9 @@ void xor_arm64_neon_2(unsigned long bytes, unsigned long *p1, 
	} while (--lines > 0);

}



void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1,

	unsigned long *p2, unsigned long *p3)

void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,

	const unsigned long * __restrict p2,

	const unsigned long * __restrict p3)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;
@@ -72,8 +73,10 @@ void xor_arm64_neon_3(unsigned long bytes, unsigned long *p1, 
	} while (--lines > 0);

}



void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1,

	unsigned long *p2, unsigned long *p3, unsigned long *p4)

void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,

	const unsigned long * __restrict p2,

	const unsigned long * __restrict p3,

	const unsigned long * __restrict p4)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;
@@ -115,9 +118,11 @@ void xor_arm64_neon_4(unsigned long bytes, unsigned long *p1, 
	} while (--lines > 0);

}



void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1,

	unsigned long *p2, unsigned long *p3,

	unsigned long *p4, unsigned long *p5)

void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,

	const unsigned long * __restrict p2,

	const unsigned long * __restrict p3,

	const unsigned long * __restrict p4,

	const unsigned long * __restrict p5)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;
@@ -186,8 +191,10 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) 
	return res;

}



static void xor_arm64_eor3_3(unsigned long bytes, unsigned long *p1,

			     unsigned long *p2, unsigned long *p3)

static void xor_arm64_eor3_3(unsigned long bytes,

	unsigned long * __restrict p1,

	const unsigned long * __restrict p2,

	const unsigned long * __restrict p3)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;
@@ -219,9 +226,11 @@ static void xor_arm64_eor3_3(unsigned long bytes, unsigned long *p1, 
	} while (--lines > 0);

}



static void xor_arm64_eor3_4(unsigned long bytes, unsigned long *p1,

			     unsigned long *p2, unsigned long *p3,

			     unsigned long *p4)

static void xor_arm64_eor3_4(unsigned long bytes,

	unsigned long * __restrict p1,

	const unsigned long * __restrict p2,

	const unsigned long * __restrict p3,

	const unsigned long * __restrict p4)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;
@@ -261,9 +270,12 @@ static void xor_arm64_eor3_4(unsigned long bytes, unsigned long *p1, 
	} while (--lines > 0);

}



static void xor_arm64_eor3_5(unsigned long bytes, unsigned long *p1,

			     unsigned long *p2, unsigned long *p3,

			     unsigned long *p4, unsigned long *p5)

static void xor_arm64_eor3_5(unsigned long bytes,

	unsigned long * __restrict p1,

	const unsigned long * __restrict p2,

	const unsigned long * __restrict p3,

	const unsigned long * __restrict p4,

	const unsigned long * __restrict p5)

{

	uint64_t *dp1 = (uint64_t *)p1;

	uint64_t *dp2 = (uint64_t *)p2;

arch/ia64/include/asm/xor.h

+14 −7
Original line number Diff line number Diff line
@@ -4,13 +4,20 @@ 
 */





extern void xor_ia64_2(unsigned long, unsigned long *, unsigned long *);

extern void xor_ia64_3(unsigned long, unsigned long *, unsigned long *,

		       unsigned long *);

extern void xor_ia64_4(unsigned long, unsigned long *, unsigned long *,

		       unsigned long *, unsigned long *);

extern void xor_ia64_5(unsigned long, unsigned long *, unsigned long *,

		       unsigned long *, unsigned long *, unsigned long *);

extern void xor_ia64_2(unsigned long bytes, unsigned long * __restrict p1,

		       const unsigned long * __restrict p2);

extern void xor_ia64_3(unsigned long bytes, unsigned long * __restrict p1,

		       const unsigned long * __restrict p2,

		       const unsigned long * __restrict p3);

extern void xor_ia64_4(unsigned long bytes, unsigned long * __restrict p1,

		       const unsigned long * __restrict p2,

		       const unsigned long * __restrict p3,

		       const unsigned long * __restrict p4);

extern void xor_ia64_5(unsigned long bytes, unsigned long * __restrict p1,

		       const unsigned long * __restrict p2,

		       const unsigned long * __restrict p3,

		       const unsigned long * __restrict p4,

		       const unsigned long * __restrict p5);



static struct xor_block_template xor_block_ia64 = {

	.name =	"ia64",