Unverified Commit 2ce5729f authored by Charlie Jenkins's avatar Charlie Jenkins Committed by Palmer Dabbelt
Browse files

riscv: Add static key for misaligned accesses 



Support static branches depending on the value of misaligned accesses.
This will be used by a later patch in the series. At any point in time,
this static branch will only be enabled if all online CPUs are
considered "fast".

Signed-off-by: default avatarCharlie Jenkins <charlie@rivosinc.com>
Reviewed-by: default avatarEvan Green <evan@rivosinc.com>
Link: https://lore.kernel.org/r/20240108-optimize_checksum-v15-2-1c50de5f2167@rivosinc.com


Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent 1e7196fa

arch/riscv/include/asm/cpufeature.h

+2 −0
Original line number Diff line number Diff line
@@ -133,4 +133,6 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi 
	return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);

}



DECLARE_STATIC_KEY_FALSE(fast_misaligned_access_speed_key);



#endif

arch/riscv/kernel/cpufeature.c

+87 −3
Original line number Diff line number Diff line
@@ -8,8 +8,10 @@ 


#include <linux/acpi.h>

#include <linux/bitmap.h>

#include <linux/cpu.h>

#include <linux/cpuhotplug.h>

#include <linux/ctype.h>

#include <linux/jump_label.h>

#include <linux/log2.h>

#include <linux/memory.h>

#include <linux/module.h>
@@ -44,6 +46,8 @@ struct riscv_isainfo hart_isa[NR_CPUS]; 
/* Performance information */

DEFINE_PER_CPU(long, misaligned_access_speed);



static cpumask_t fast_misaligned_access;



/**

 * riscv_isa_extension_base() - Get base extension word

 *
@@ -643,6 +647,16 @@ static int check_unaligned_access(void *param) 
		(speed == RISCV_HWPROBE_MISALIGNED_FAST) ? "fast" : "slow");



	per_cpu(misaligned_access_speed, cpu) = speed;



	/*

	 * Set the value of fast_misaligned_access of a CPU. These operations

	 * are atomic to avoid race conditions.

	 */

	if (speed == RISCV_HWPROBE_MISALIGNED_FAST)

		cpumask_set_cpu(cpu, &fast_misaligned_access);

	else

		cpumask_clear_cpu(cpu, &fast_misaligned_access);



	return 0;

}
@@ -655,13 +669,69 @@ static void check_unaligned_access_nonboot_cpu(void *param) 
		check_unaligned_access(pages[cpu]);

}



DEFINE_STATIC_KEY_FALSE(fast_misaligned_access_speed_key);



static void modify_unaligned_access_branches(cpumask_t *mask, int weight)

{

	if (cpumask_weight(mask) == weight)

		static_branch_enable_cpuslocked(&fast_misaligned_access_speed_key);

	else

		static_branch_disable_cpuslocked(&fast_misaligned_access_speed_key);

}



static void set_unaligned_access_static_branches_except_cpu(int cpu)

{

	/*

	 * Same as set_unaligned_access_static_branches, except excludes the

	 * given CPU from the result. When a CPU is hotplugged into an offline

	 * state, this function is called before the CPU is set to offline in

	 * the cpumask, and thus the CPU needs to be explicitly excluded.

	 */



	cpumask_t fast_except_me;



	cpumask_and(&fast_except_me, &fast_misaligned_access, cpu_online_mask);

	cpumask_clear_cpu(cpu, &fast_except_me);



	modify_unaligned_access_branches(&fast_except_me, num_online_cpus() - 1);

}



static void set_unaligned_access_static_branches(void)

{

	/*

	 * This will be called after check_unaligned_access_all_cpus so the

	 * result of unaligned access speed for all CPUs will be available.

	 *

	 * To avoid the number of online cpus changing between reading

	 * cpu_online_mask and calling num_online_cpus, cpus_read_lock must be

	 * held before calling this function.

	 */



	cpumask_t fast_and_online;



	cpumask_and(&fast_and_online, &fast_misaligned_access, cpu_online_mask);



	modify_unaligned_access_branches(&fast_and_online, num_online_cpus());

}



static int lock_and_set_unaligned_access_static_branch(void)

{

	cpus_read_lock();

	set_unaligned_access_static_branches();

	cpus_read_unlock();



	return 0;

}



arch_initcall_sync(lock_and_set_unaligned_access_static_branch);



static int riscv_online_cpu(unsigned int cpu)

{

	static struct page *buf;



	/* We are already set since the last check */

	if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_UNKNOWN)

		return 0;

		goto exit;



	buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);

	if (!buf) {
@@ -671,6 +741,17 @@ static int riscv_online_cpu(unsigned int cpu) 


	check_unaligned_access(buf);

	__free_pages(buf, MISALIGNED_BUFFER_ORDER);



exit:

	set_unaligned_access_static_branches();



	return 0;

}



static int riscv_offline_cpu(unsigned int cpu)

{

	set_unaligned_access_static_branches_except_cpu(cpu);



	return 0;

}
@@ -705,9 +786,12 @@ static int check_unaligned_access_all_cpus(void) 
	/* Check core 0. */

	smp_call_on_cpu(0, check_unaligned_access, bufs[0], true);



	/* Setup hotplug callback for any new CPUs that come online. */

	/*

	 * Setup hotplug callbacks for any new CPUs that come online or go

	 * offline.

	 */

	cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",

				  riscv_online_cpu, NULL);

				  riscv_online_cpu, riscv_offline_cpu);



out:

	unaligned_emulation_finish();