Commit 7c679286 authored by Heiko Carstens's avatar Heiko Carstens Committed by Alexander Gordeev
Browse files

s390: Provide optimized __arch_hweight*() implementations 



Make use of the popcnt instruction to provide optimized __arch_hweight*()
implementations. The generated code is shorter and avoids rather expensive
functions calls.

Acked-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
Signed-off-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
parent 447360d7

arch/s390/Kconfig

+1 −1
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ config ARCH_PROC_KCORE_TEXT 
	def_bool y



config GENERIC_HWEIGHT

	def_bool y

	def_bool !HAVE_MARCH_Z196_FEATURES



config GENERIC_BUG

	def_bool y if BUG

arch/s390/include/asm/arch_hweight.h

0 → 100644
+76 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#ifndef _ASM_S390_ARCH_HWEIGHT_H

#define _ASM_S390_ARCH_HWEIGHT_H



#include <linux/types.h>



static __always_inline unsigned long popcnt_z196(unsigned long w)

{

	unsigned long cnt;



	asm volatile(".insn	rrf,0xb9e10000,%[cnt],%[w],0,0"

		     : [cnt] "=d" (cnt)

		     : [w] "d" (w)

		     : "cc");

	return cnt;

}



static __always_inline unsigned long popcnt_z15(unsigned long w)

{

	unsigned long cnt;



	asm volatile(".insn	rrf,0xb9e10000,%[cnt],%[w],8,0"

		     : [cnt] "=d" (cnt)

		     : [w] "d" (w)

		     : "cc");

	return cnt;

}



static __always_inline unsigned long __arch_hweight64(__u64 w)

{

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES))

		return popcnt_z15(w);

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) {

		w = popcnt_z196(w);

		w += w >> 32;

		w += w >> 16;

		w += w >> 8;

		return w & 0xff;

	}

	return __sw_hweight64(w);

}



static __always_inline unsigned int __arch_hweight32(unsigned int w)

{

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES))

		return popcnt_z15(w);

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) {

		w = popcnt_z196(w);

		w += w >> 16;

		w += w >> 8;

		return w & 0xff;

	}

	return __sw_hweight32(w);

}



static __always_inline unsigned int __arch_hweight16(unsigned int w)

{

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z15_FEATURES))

		return popcnt_z15((unsigned short)w);

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES)) {

		w = popcnt_z196(w);

		w += w >> 8;

		return w & 0xff;

	}

	return __sw_hweight16(w);

}



static __always_inline unsigned int __arch_hweight8(unsigned int w)

{

	if (IS_ENABLED(CONFIG_HAVE_MARCH_Z196_FEATURES))

		return popcnt_z196((unsigned char)w);

	return __sw_hweight8(w);

}



#endif /* _ASM_S390_ARCH_HWEIGHT_H */

arch/s390/include/asm/bitops.h

+2 −1
Original line number Diff line number Diff line
@@ -379,8 +379,9 @@ static inline int fls(unsigned int word) 
	return fls64(word);

}



#include <asm/arch_hweight.h>

#include <asm-generic/bitops/const_hweight.h>

#include <asm-generic/bitops/ffz.h>

#include <asm-generic/bitops/hweight.h>

#include <asm-generic/bitops/sched.h>

#include <asm-generic/bitops/le.h>

#include <asm-generic/bitops/ext2-atomic-setbit.h>