Unverified Commit c6408684 authored by Palmer Dabbelt's avatar Palmer Dabbelt
Browse files

Merge patch series "riscv: Add fine-tuned checksum functions" 

Charlie Jenkins <charlie@rivosinc.com> says:

Each architecture generally implements fine-tuned checksum functions to
leverage the instruction set. This patch adds the main checksum
functions that are used in networking. Tested on QEMU, this series
allows the CHECKSUM_KUNIT tests to complete an average of 50.9% faster.

This patch takes heavy use of the Zbb extension using alternatives
patching.

To test this patch, enable the configs for KUNIT, then CHECKSUM_KUNIT.

I have attempted to make these functions as optimal as possible, but I
have not ran anything on actual riscv hardware. My performance testing
has been limited to inspecting the assembly, running the algorithms on
x86 hardware, and running in QEMU.

ip_fast_csum is a relatively small function so even though it is
possible to read 64 bits at a time on compatible hardware, the
bottleneck becomes the clean up and setup code so loading 32 bits at a
time is actually faster.

* b4-shazam-merge:
  kunit: Add tests for csum_ipv6_magic and ip_fast_csum
  riscv: Add checksum library
  riscv: Add checksum header
  riscv: Add static key for misaligned accesses
  asm-generic: Improve csum_fold

Link: https://lore.kernel.org/r/20240108-optimize_checksum-v15-0-1c50de5f2167@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parents 0de65288 6f4c45cb

arch/riscv/include/asm/checksum.h

0 → 100644
+93 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * Checksum routines

 *

 * Copyright (C) 2023 Rivos Inc.

 */

#ifndef __ASM_RISCV_CHECKSUM_H

#define __ASM_RISCV_CHECKSUM_H



#include <linux/in6.h>

#include <linux/uaccess.h>



#define ip_fast_csum ip_fast_csum



extern unsigned int do_csum(const unsigned char *buff, int len);

#define do_csum do_csum



/* Default version is sufficient for 32 bit */

#ifndef CONFIG_32BIT

#define _HAVE_ARCH_IPV6_CSUM

__sum16 csum_ipv6_magic(const struct in6_addr *saddr,

			const struct in6_addr *daddr,

			__u32 len, __u8 proto, __wsum sum);

#endif



/* Define riscv versions of functions before importing asm-generic/checksum.h */

#include <asm-generic/checksum.h>



/**

 * Quickly compute an IP checksum with the assumption that IPv4 headers will

 * always be in multiples of 32-bits, and have an ihl of at least 5.

 *

 * @ihl: the number of 32 bit segments and must be greater than or equal to 5.

 * @iph: assumed to be word aligned given that NET_IP_ALIGN is set to 2 on

 *  riscv, defining IP headers to be aligned.

 */

static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)

{

	unsigned long csum = 0;

	int pos = 0;



	do {

		csum += ((const unsigned int *)iph)[pos];

		if (IS_ENABLED(CONFIG_32BIT))

			csum += csum < ((const unsigned int *)iph)[pos];

	} while (++pos < ihl);



	/*

	 * ZBB only saves three instructions on 32-bit and five on 64-bit so not

	 * worth checking if supported without Alternatives.

	 */

	if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&

	    IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {

		unsigned long fold_temp;



		asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,

					      RISCV_ISA_EXT_ZBB, 1)

		    :

		    :

		    :

		    : no_zbb);



		if (IS_ENABLED(CONFIG_32BIT)) {

			asm(".option push				\n\

			.option arch,+zbb				\n\

				not	%[fold_temp], %[csum]		\n\

				rori	%[csum], %[csum], 16		\n\

				sub	%[csum], %[fold_temp], %[csum]	\n\

			.option pop"

			: [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp));

		} else {

			asm(".option push				\n\

			.option arch,+zbb				\n\

				rori	%[fold_temp], %[csum], 32	\n\

				add	%[csum], %[fold_temp], %[csum]	\n\

				srli	%[csum], %[csum], 32		\n\

				not	%[fold_temp], %[csum]		\n\

				roriw	%[csum], %[csum], 16		\n\

				subw	%[csum], %[fold_temp], %[csum]	\n\

			.option pop"

			: [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp));

		}

		return (__force __sum16)(csum >> 16);

	}

no_zbb:

#ifndef CONFIG_32BIT

	csum += ror64(csum, 32);

	csum >>= 32;

#endif

	return csum_fold((__force __wsum)csum);

}



#endif /* __ASM_RISCV_CHECKSUM_H */

arch/riscv/include/asm/cpufeature.h

+2 −0
Original line number Diff line number Diff line
@@ -135,4 +135,6 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi 
	return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);

}



DECLARE_STATIC_KEY_FALSE(fast_misaligned_access_speed_key);



#endif

arch/riscv/kernel/cpufeature.c

+87 −3
Original line number Diff line number Diff line
@@ -8,8 +8,10 @@ 


#include <linux/acpi.h>

#include <linux/bitmap.h>

#include <linux/cpu.h>

#include <linux/cpuhotplug.h>

#include <linux/ctype.h>

#include <linux/jump_label.h>

#include <linux/log2.h>

#include <linux/memory.h>

#include <linux/module.h>
@@ -44,6 +46,8 @@ struct riscv_isainfo hart_isa[NR_CPUS]; 
/* Performance information */

DEFINE_PER_CPU(long, misaligned_access_speed);



static cpumask_t fast_misaligned_access;



/**

 * riscv_isa_extension_base() - Get base extension word

 *
@@ -784,6 +788,16 @@ static int check_unaligned_access(void *param) 
		(speed == RISCV_HWPROBE_MISALIGNED_FAST) ? "fast" : "slow");



	per_cpu(misaligned_access_speed, cpu) = speed;



	/*

	 * Set the value of fast_misaligned_access of a CPU. These operations

	 * are atomic to avoid race conditions.

	 */

	if (speed == RISCV_HWPROBE_MISALIGNED_FAST)

		cpumask_set_cpu(cpu, &fast_misaligned_access);

	else

		cpumask_clear_cpu(cpu, &fast_misaligned_access);



	return 0;

}
@@ -796,13 +810,69 @@ static void check_unaligned_access_nonboot_cpu(void *param) 
		check_unaligned_access(pages[cpu]);

}



DEFINE_STATIC_KEY_FALSE(fast_misaligned_access_speed_key);



static void modify_unaligned_access_branches(cpumask_t *mask, int weight)

{

	if (cpumask_weight(mask) == weight)

		static_branch_enable_cpuslocked(&fast_misaligned_access_speed_key);

	else

		static_branch_disable_cpuslocked(&fast_misaligned_access_speed_key);

}



static void set_unaligned_access_static_branches_except_cpu(int cpu)

{

	/*

	 * Same as set_unaligned_access_static_branches, except excludes the

	 * given CPU from the result. When a CPU is hotplugged into an offline

	 * state, this function is called before the CPU is set to offline in

	 * the cpumask, and thus the CPU needs to be explicitly excluded.

	 */



	cpumask_t fast_except_me;



	cpumask_and(&fast_except_me, &fast_misaligned_access, cpu_online_mask);

	cpumask_clear_cpu(cpu, &fast_except_me);



	modify_unaligned_access_branches(&fast_except_me, num_online_cpus() - 1);

}



static void set_unaligned_access_static_branches(void)

{

	/*

	 * This will be called after check_unaligned_access_all_cpus so the

	 * result of unaligned access speed for all CPUs will be available.

	 *

	 * To avoid the number of online cpus changing between reading

	 * cpu_online_mask and calling num_online_cpus, cpus_read_lock must be

	 * held before calling this function.

	 */



	cpumask_t fast_and_online;



	cpumask_and(&fast_and_online, &fast_misaligned_access, cpu_online_mask);



	modify_unaligned_access_branches(&fast_and_online, num_online_cpus());

}



static int lock_and_set_unaligned_access_static_branch(void)

{

	cpus_read_lock();

	set_unaligned_access_static_branches();

	cpus_read_unlock();



	return 0;

}



arch_initcall_sync(lock_and_set_unaligned_access_static_branch);



static int riscv_online_cpu(unsigned int cpu)

{

	static struct page *buf;



	/* We are already set since the last check */

	if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_UNKNOWN)

		return 0;

		goto exit;



	buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);

	if (!buf) {
@@ -812,6 +882,17 @@ static int riscv_online_cpu(unsigned int cpu) 


	check_unaligned_access(buf);

	__free_pages(buf, MISALIGNED_BUFFER_ORDER);



exit:

	set_unaligned_access_static_branches();



	return 0;

}



static int riscv_offline_cpu(unsigned int cpu)

{

	set_unaligned_access_static_branches_except_cpu(cpu);



	return 0;

}
@@ -846,9 +927,12 @@ static int check_unaligned_access_all_cpus(void) 
	/* Check core 0. */

	smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);



	/* Setup hotplug callback for any new CPUs that come online. */

	/*

	 * Setup hotplug callbacks for any new CPUs that come online or go

	 * offline.

	 */

	cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",

				  riscv_online_cpu, NULL);

				  riscv_online_cpu, riscv_offline_cpu);



out:

	unaligned_emulation_finish();

arch/riscv/lib/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -6,10 +6,11 @@ lib-y += memmove.o 
lib-y			+= strcmp.o

lib-y			+= strlen.o

lib-y			+= strncmp.o

lib-y			+= csum.o

ifeq ($(CONFIG_MMU), y)

lib-y				+= uaccess.o

lib-$(CONFIG_RISCV_ISA_V)	+= uaccess_vector.o

endif

lib-$(CONFIG_MMU)	+= uaccess.o

lib-$(CONFIG_64BIT)	+= tishift.o

lib-$(CONFIG_RISCV_ISA_ZICBOZ)	+= clear_page.o

arch/riscv/lib/csum.c

0 → 100644
+326 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * Checksum library

 *

 * Influenced by arch/arm64/lib/csum.c

 * Copyright (C) 2023 Rivos Inc.

 */

#include <linux/bitops.h>

#include <linux/compiler.h>

#include <linux/jump_label.h>

#include <linux/kasan-checks.h>

#include <linux/kernel.h>



#include <asm/cpufeature.h>



#include <net/checksum.h>



/* Default version is sufficient for 32 bit */

#ifndef CONFIG_32BIT

__sum16 csum_ipv6_magic(const struct in6_addr *saddr,

			const struct in6_addr *daddr,

			__u32 len, __u8 proto, __wsum csum)

{

	unsigned int ulen, uproto;

	unsigned long sum = (__force unsigned long)csum;



	sum += (__force unsigned long)saddr->s6_addr32[0];

	sum += (__force unsigned long)saddr->s6_addr32[1];

	sum += (__force unsigned long)saddr->s6_addr32[2];

	sum += (__force unsigned long)saddr->s6_addr32[3];



	sum += (__force unsigned long)daddr->s6_addr32[0];

	sum += (__force unsigned long)daddr->s6_addr32[1];

	sum += (__force unsigned long)daddr->s6_addr32[2];

	sum += (__force unsigned long)daddr->s6_addr32[3];



	ulen = (__force unsigned int)htonl((unsigned int)len);

	sum += ulen;



	uproto = (__force unsigned int)htonl(proto);

	sum += uproto;



	/*

	 * Zbb support saves 4 instructions, so not worth checking without

	 * alternatives if supported

	 */

	if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&

	    IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {

		unsigned long fold_temp;



		/*

		 * Zbb is likely available when the kernel is compiled with Zbb

		 * support, so nop when Zbb is available and jump when Zbb is

		 * not available.

		 */

		asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,

					      RISCV_ISA_EXT_ZBB, 1)

				  :

				  :

				  :

				  : no_zbb);

		asm(".option push					\n\

		.option arch,+zbb					\n\

			rori	%[fold_temp], %[sum], 32		\n\

			add	%[sum], %[fold_temp], %[sum]		\n\

			srli	%[sum], %[sum], 32			\n\

			not	%[fold_temp], %[sum]			\n\

			roriw	%[sum], %[sum], 16			\n\

			subw	%[sum], %[fold_temp], %[sum]		\n\

		.option pop"

		: [sum] "+r" (sum), [fold_temp] "=&r" (fold_temp));

		return (__force __sum16)(sum >> 16);

	}

no_zbb:

	sum += ror64(sum, 32);

	sum >>= 32;

	return csum_fold((__force __wsum)sum);

}

EXPORT_SYMBOL(csum_ipv6_magic);

#endif /* !CONFIG_32BIT */



#ifdef CONFIG_32BIT

#define OFFSET_MASK 3

#elif CONFIG_64BIT

#define OFFSET_MASK 7

#endif



static inline __no_sanitize_address unsigned long

do_csum_common(const unsigned long *ptr, const unsigned long *end,

	       unsigned long data)

{

	unsigned int shift;

	unsigned long csum = 0, carry = 0;



	/*

	 * Do 32-bit reads on RV32 and 64-bit reads otherwise. This should be

	 * faster than doing 32-bit reads on architectures that support larger

	 * reads.

	 */

	while (ptr < end) {

		csum += data;

		carry += csum < data;

		data = *(ptr++);

	}



	/*

	 * Perform alignment (and over-read) bytes on the tail if any bytes

	 * leftover.

	 */

	shift = ((long)ptr - (long)end) * 8;

#ifdef __LITTLE_ENDIAN

	data = (data << shift) >> shift;

#else

	data = (data >> shift) << shift;

#endif

	csum += data;

	carry += csum < data;

	csum += carry;

	csum += csum < carry;



	return csum;

}



/*

 * Algorithm accounts for buff being misaligned.

 * If buff is not aligned, will over-read bytes but not use the bytes that it

 * shouldn't. The same thing will occur on the tail-end of the read.

 */

static inline __no_sanitize_address unsigned int

do_csum_with_alignment(const unsigned char *buff, int len)

{

	unsigned int offset, shift;

	unsigned long csum, data;

	const unsigned long *ptr, *end;



	/*

	 * Align address to closest word (double word on rv64) that comes before

	 * buff. This should always be in the same page and cache line.

	 * Directly call KASAN with the alignment we will be using.

	 */

	offset = (unsigned long)buff & OFFSET_MASK;

	kasan_check_read(buff, len);

	ptr = (const unsigned long *)(buff - offset);



	/*

	 * Clear the most significant bytes that were over-read if buff was not

	 * aligned.

	 */

	shift = offset * 8;

	data = *(ptr++);

#ifdef __LITTLE_ENDIAN

	data = (data >> shift) << shift;

#else

	data = (data << shift) >> shift;

#endif

	end = (const unsigned long *)(buff + len);

	csum = do_csum_common(ptr, end, data);



	/*

	 * Zbb support saves 6 instructions, so not worth checking without

	 * alternatives if supported

	 */

	if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&

	    IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {

		unsigned long fold_temp;



		/*

		 * Zbb is likely available when the kernel is compiled with Zbb

		 * support, so nop when Zbb is available and jump when Zbb is

		 * not available.

		 */

		asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,

					      RISCV_ISA_EXT_ZBB, 1)

				  :

				  :

				  :

				  : no_zbb);



#ifdef CONFIG_32BIT

		asm_volatile_goto(".option push			\n\

		.option arch,+zbb				\n\

			rori	%[fold_temp], %[csum], 16	\n\

			andi	%[offset], %[offset], 1		\n\

			add	%[csum], %[fold_temp], %[csum]	\n\

			beq	%[offset], zero, %l[end]	\n\

			rev8	%[csum], %[csum]		\n\

		.option pop"

			: [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp)

			: [offset] "r" (offset)

			:

			: end);



		return (unsigned short)csum;

#else /* !CONFIG_32BIT */

		asm_volatile_goto(".option push			\n\

		.option arch,+zbb				\n\

			rori	%[fold_temp], %[csum], 32	\n\

			add	%[csum], %[fold_temp], %[csum]	\n\

			srli	%[csum], %[csum], 32		\n\

			roriw	%[fold_temp], %[csum], 16	\n\

			addw	%[csum], %[fold_temp], %[csum]	\n\

			andi	%[offset], %[offset], 1		\n\

			beq	%[offset], zero, %l[end]	\n\

			rev8	%[csum], %[csum]		\n\

		.option pop"

			: [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp)

			: [offset] "r" (offset)

			:

			: end);



		return (csum << 16) >> 48;

#endif /* !CONFIG_32BIT */

end:

		return csum >> 16;

	}

no_zbb:

#ifndef CONFIG_32BIT

	csum += ror64(csum, 32);

	csum >>= 32;

#endif

	csum = (u32)csum + ror32((u32)csum, 16);

	if (offset & 1)

		return (u16)swab32(csum);

	return csum >> 16;

}



/*

 * Does not perform alignment, should only be used if machine has fast

 * misaligned accesses, or when buff is known to be aligned.

 */

static inline __no_sanitize_address unsigned int

do_csum_no_alignment(const unsigned char *buff, int len)

{

	unsigned long csum, data;

	const unsigned long *ptr, *end;



	ptr = (const unsigned long *)(buff);

	data = *(ptr++);



	kasan_check_read(buff, len);



	end = (const unsigned long *)(buff + len);

	csum = do_csum_common(ptr, end, data);



	/*

	 * Zbb support saves 6 instructions, so not worth checking without

	 * alternatives if supported

	 */

	if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) &&

	    IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {

		unsigned long fold_temp;



		/*

		 * Zbb is likely available when the kernel is compiled with Zbb

		 * support, so nop when Zbb is available and jump when Zbb is

		 * not available.

		 */

		asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0,

					      RISCV_ISA_EXT_ZBB, 1)

				  :

				  :

				  :

				  : no_zbb);



#ifdef CONFIG_32BIT

		asm (".option push				\n\

		.option arch,+zbb				\n\

			rori	%[fold_temp], %[csum], 16	\n\

			add	%[csum], %[fold_temp], %[csum]	\n\

		.option pop"

			: [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp)

			:

			: );



#else /* !CONFIG_32BIT */

		asm (".option push				\n\

		.option arch,+zbb				\n\

			rori	%[fold_temp], %[csum], 32	\n\

			add	%[csum], %[fold_temp], %[csum]	\n\

			srli	%[csum], %[csum], 32		\n\

			roriw	%[fold_temp], %[csum], 16	\n\

			addw	%[csum], %[fold_temp], %[csum]	\n\

		.option pop"

			: [csum] "+r" (csum), [fold_temp] "=&r" (fold_temp)

			:

			: );

#endif /* !CONFIG_32BIT */

		return csum >> 16;

	}

no_zbb:

#ifndef CONFIG_32BIT

	csum += ror64(csum, 32);

	csum >>= 32;

#endif

	csum = (u32)csum + ror32((u32)csum, 16);

	return csum >> 16;

}



/*

 * Perform a checksum on an arbitrary memory address.

 * Will do a light-weight address alignment if buff is misaligned, unless

 * cpu supports fast misaligned accesses.

 */

unsigned int do_csum(const unsigned char *buff, int len)

{

	if (unlikely(len <= 0))

		return 0;



	/*

	 * Significant performance gains can be seen by not doing alignment

	 * on machines with fast misaligned accesses.

	 *

	 * There is some duplicate code between the "with_alignment" and

	 * "no_alignment" implmentations, but the overlap is too awkward to be

	 * able to fit in one function without introducing multiple static

	 * branches. The largest chunk of overlap was delegated into the

	 * do_csum_common function.

	 */

	if (static_branch_likely(&fast_misaligned_access_speed_key))

		return do_csum_no_alignment(buff, len);



	if (((unsigned long)buff & OFFSET_MASK) == 0)

		return do_csum_no_alignment(buff, len);



	return do_csum_with_alignment(buff, len);

}