Commit 9c44575c authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'bitmap-for-6.12' of https://github.com/norov/linux 

Pull bitmap updates from Yury Norov:

 - switch all bitmamp APIs from inline to __always_inline (Brian Norris)

   The __always_inline series improves on code generation, and now with
   the latest compiler versions is required to avoid compilation
   warnings. It spent enough in my backlog, and I'm thankful to Brian
   Norris for taking over and moving it forward.

 - introduce GENMASK_U128() macro (Anshuman Khandual)

   GENMASK_U128() is a prerequisite needed for arm64 development

* tag 'bitmap-for-6.12' of https://github.com/norov/linux:
  lib/test_bits.c: Add tests for GENMASK_U128()
  uapi: Define GENMASK_U128
  nodemask: Switch from inline to __always_inline
  cpumask: Switch from inline to __always_inline
  bitmap: Switch from inline to __always_inline
  find: Switch from inline to __always_inline
parents ba33a49f d7bcc374

include/linux/bitmap.h

+76 −64
Original line number Diff line number Diff line
@@ -203,8 +203,8 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, 
 * the bit offset of all zero areas this function finds is multiples of that

 * power of 2. A @align_mask of 0 means no alignment is required.

 */

static inline unsigned long

bitmap_find_next_zero_area(unsigned long *map,

static __always_inline

unsigned long bitmap_find_next_zero_area(unsigned long *map,

					 unsigned long size,

					 unsigned long start,

					 unsigned int nr,
@@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, 


#define bitmap_size(nbits)	(ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)



static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)

static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)

{

	unsigned int len = bitmap_size(nbits);
@@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) 
		memset(dst, 0, len);

}



static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)

static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)

{

	unsigned int len = bitmap_size(nbits);
@@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) 
		memset(dst, 0xff, len);

}



static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,

			unsigned int nbits)

static __always_inline

void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits)

{

	unsigned int len = bitmap_size(nbits);
@@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, 
/*

 * Copy bitmap and clear tail bits in last word.

 */

static inline void bitmap_copy_clear_tail(unsigned long *dst,

		const unsigned long *src, unsigned int nbits)

static __always_inline

void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits)

{

	bitmap_copy(dst, src, nbits);

	if (nbits % BITS_PER_LONG)
@@ -318,7 +318,8 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); 
	bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))

#endif



static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,

static __always_inline

bool bitmap_and(unsigned long *dst, const unsigned long *src1,

		const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))
@@ -326,7 +327,8 @@ static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, 
	return __bitmap_and(dst, src1, src2, nbits);

}



static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,

static __always_inline

void bitmap_or(unsigned long *dst, const unsigned long *src1,

	       const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))
@@ -335,7 +337,8 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, 
		__bitmap_or(dst, src1, src2, nbits);

}



static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,

static __always_inline

void bitmap_xor(unsigned long *dst, const unsigned long *src1,

		const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))
@@ -344,7 +347,8 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, 
		__bitmap_xor(dst, src1, src2, nbits);

}



static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,

static __always_inline

bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,

		   const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))
@@ -352,8 +356,8 @@ static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, 
	return __bitmap_andnot(dst, src1, src2, nbits);

}



static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,

			unsigned int nbits)

static __always_inline

void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits)

{

	if (small_const_nbits(nbits))

		*dst = ~(*src);
@@ -368,8 +372,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr 
#endif

#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)



static inline bool bitmap_equal(const unsigned long *src1,

				const unsigned long *src2, unsigned int nbits)

static __always_inline

bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))

		return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
@@ -388,10 +392,9 @@ static inline bool bitmap_equal(const unsigned long *src1, 
 *

 * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise

 */

static inline bool bitmap_or_equal(const unsigned long *src1,

				   const unsigned long *src2,

				   const unsigned long *src3,

				   unsigned int nbits)

static __always_inline

bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2,

		     const unsigned long *src3, unsigned int nbits)

{

	if (!small_const_nbits(nbits))

		return __bitmap_or_equal(src1, src2, src3, nbits);
@@ -399,9 +402,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1, 
	return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));

}



static inline bool bitmap_intersects(const unsigned long *src1,

				     const unsigned long *src2,

				     unsigned int nbits)

static __always_inline

bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))

		return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
@@ -409,8 +411,8 @@ static inline bool bitmap_intersects(const unsigned long *src1, 
		return __bitmap_intersects(src1, src2, nbits);

}



static inline bool bitmap_subset(const unsigned long *src1,

				 const unsigned long *src2, unsigned int nbits)

static __always_inline

bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)

{

	if (small_const_nbits(nbits))

		return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
@@ -418,7 +420,8 @@ static inline bool bitmap_subset(const unsigned long *src1, 
		return __bitmap_subset(src1, src2, nbits);

}



static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)

static __always_inline

bool bitmap_empty(const unsigned long *src, unsigned nbits)

{

	if (small_const_nbits(nbits))

		return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -426,7 +429,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) 
	return find_first_bit(src, nbits) == nbits;

}



static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)

static __always_inline

bool bitmap_full(const unsigned long *src, unsigned int nbits)

{

	if (small_const_nbits(nbits))

		return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@@ -460,8 +464,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, 
	return __bitmap_weight_andnot(src1, src2, nbits);

}



static __always_inline void bitmap_set(unsigned long *map, unsigned int start,

		unsigned int nbits)

static __always_inline

void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)

{

	if (__builtin_constant_p(nbits) && nbits == 1)

		__set_bit(start, map);
@@ -476,8 +480,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, 
		__bitmap_set(map, start, nbits);

}



static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,

		unsigned int nbits)

static __always_inline

void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)

{

	if (__builtin_constant_p(nbits) && nbits == 1)

		__clear_bit(start, map);
@@ -492,7 +496,8 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, 
		__bitmap_clear(map, start, nbits);

}



static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,

static __always_inline

void bitmap_shift_right(unsigned long *dst, const unsigned long *src,

			unsigned int shift, unsigned int nbits)

{

	if (small_const_nbits(nbits))
@@ -501,7 +506,8 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s 
		__bitmap_shift_right(dst, src, shift, nbits);

}



static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,

static __always_inline

void bitmap_shift_left(unsigned long *dst, const unsigned long *src,

		       unsigned int shift, unsigned int nbits)

{

	if (small_const_nbits(nbits))
@@ -510,7 +516,8 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr 
		__bitmap_shift_left(dst, src, shift, nbits);

}



static inline void bitmap_replace(unsigned long *dst,

static __always_inline

void bitmap_replace(unsigned long *dst,

		    const unsigned long *old,

		    const unsigned long *new,

		    const unsigned long *mask,
@@ -557,7 +564,8 @@ static inline void bitmap_replace(unsigned long *dst, 
 * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.

 * See bitmap_scatter() for details related to this relationship.

 */

static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,

static __always_inline

void bitmap_scatter(unsigned long *dst, const unsigned long *src,

		    const unsigned long *mask, unsigned int nbits)

{

	unsigned int n = 0;
@@ -611,7 +619,8 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, 
 * bitmap_scatter(res, src, mask, n) and a call to

 * bitmap_scatter(res, result, mask, n) will lead to the same res value.

 */

static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,

static __always_inline

void bitmap_gather(unsigned long *dst, const unsigned long *src,

		   const unsigned long *mask, unsigned int nbits)

{

	unsigned int n = 0;
@@ -623,9 +632,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, 
		__assign_bit(n++, dst, test_bit(bit, src));

}



static inline void bitmap_next_set_region(unsigned long *bitmap,

					  unsigned int *rs, unsigned int *re,

					  unsigned int end)

static __always_inline

void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs,

			    unsigned int *re, unsigned int end)

{

	*rs = find_next_bit(bitmap, end, *rs);

	*re = find_next_zero_bit(bitmap, end, *rs + 1);
@@ -640,7 +649,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, 
 * This is the complement to __bitmap_find_free_region() and releases

 * the found region (by clearing it in the bitmap).

 */

static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)

static __always_inline

void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)

{

	bitmap_clear(bitmap, pos, BIT(order));

}
@@ -656,7 +666,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos 
 * Returns: 0 on success, or %-EBUSY if specified region wasn't

 * free (not all bits were zero).

 */

static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)

static __always_inline

int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)

{

	unsigned int len = BIT(order);
@@ -680,7 +691,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos 
 * Returns: the bit offset in bitmap of the allocated region,

 * or -errno on failure.

 */

static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)

static __always_inline

int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)

{

	unsigned int pos, end;		/* scans bitmap by regions of size order */
@@ -734,7 +746,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi 
 * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,

 * but we expect the lower 32-bits of u64.

 */

static inline void bitmap_from_u64(unsigned long *dst, u64 mask)

static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)

{

	bitmap_from_arr64(dst, &mask, 64);

}
@@ -749,9 +761,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) 
 * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return

 * value is undefined.

 */

static inline unsigned long bitmap_read(const unsigned long *map,

					unsigned long start,

					unsigned long nbits)

static __always_inline

unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)

{

	size_t index = BIT_WORD(start);

	unsigned long offset = start % BITS_PER_LONG;
@@ -784,7 +795,8 @@ static inline unsigned long bitmap_read(const unsigned long *map, 
 *

 * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.

 */

static inline void bitmap_write(unsigned long *map, unsigned long value,

static __always_inline

void bitmap_write(unsigned long *map, unsigned long value,

		  unsigned long start, unsigned long nbits)

{

	size_t index;

include/linux/bits.h

+15 −0
Original line number Diff line number Diff line
@@ -36,4 +36,19 @@ 
#define GENMASK_ULL(h, l) \

	(GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l))



#if !defined(__ASSEMBLY__)

/*

 * Missing asm support

 *

 * __GENMASK_U128() depends on _BIT128() which would not work

 * in the asm code, as it shifts an 'unsigned __init128' data

 * type instead of direct representation of 128 bit constants

 * such as long and unsigned long. The fundamental problem is

 * that a 128 bit constant will get silently truncated by the

 * gcc compiler.

 */

#define GENMASK_U128(h, l) \

	(GENMASK_INPUT_CHECK(h, l) + __GENMASK_U128(h, l))

#endif



#endif	/* __LINUX_BITS_H */

include/linux/cpumask.h

+112 −100

File changed.

Preview size limit exceeded, changes collapsed.

include/linux/find.h

+25 −25
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ unsigned long _find_next_bit_le(const unsigned long *addr, unsigned 
 * Returns the bit number for the next set bit

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_bit(const unsigned long *addr, unsigned long size,

			    unsigned long offset)

{
@@ -81,7 +81,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, 
 * Returns the bit number for the next set bit

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_and_bit(const unsigned long *addr1,

		const unsigned long *addr2, unsigned long size,

		unsigned long offset)
@@ -112,7 +112,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, 
 * Returns the bit number for the next set bit

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_andnot_bit(const unsigned long *addr1,

		const unsigned long *addr2, unsigned long size,

		unsigned long offset)
@@ -142,7 +142,7 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1, 
 * Returns the bit number for the next set bit

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_or_bit(const unsigned long *addr1,

		const unsigned long *addr2, unsigned long size,

		unsigned long offset)
@@ -171,7 +171,7 @@ unsigned long find_next_or_bit(const unsigned long *addr1, 
 * Returns the bit number of the next zero bit

 * If no bits are zero, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,

				 unsigned long offset)

{
@@ -198,7 +198,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, 
 * Returns the bit number of the first set bit.

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_first_bit(const unsigned long *addr, unsigned long size)

{

	if (small_const_nbits(size)) {
@@ -224,7 +224,7 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) 
 * Returns the bit number of the N'th set bit.

 * If no such, returns >= @size.

 */

static inline

static __always_inline

unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)

{

	if (n >= size)
@@ -249,7 +249,7 @@ unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsign 
 * Returns the bit number of the N'th set bit.

 * If no such, returns @size.

 */

static inline

static __always_inline

unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,

				unsigned long size, unsigned long n)

{
@@ -276,7 +276,7 @@ unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long * 
 * Returns the bit number of the N'th set bit.

 * If no such, returns @size.

 */

static inline

static __always_inline

unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,

				unsigned long size, unsigned long n)

{
@@ -332,7 +332,7 @@ unsigned long find_nth_and_andnot_bit(const unsigned long *addr1, 
 * Returns the bit number for the next set bit

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_first_and_bit(const unsigned long *addr1,

				 const unsigned long *addr2,

				 unsigned long size)
@@ -357,7 +357,7 @@ unsigned long find_first_and_bit(const unsigned long *addr1, 
 * Returns the bit number for the first set bit

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_first_and_and_bit(const unsigned long *addr1,

				     const unsigned long *addr2,

				     const unsigned long *addr3,
@@ -381,7 +381,7 @@ unsigned long find_first_and_and_bit(const unsigned long *addr1, 
 * Returns the bit number of the first cleared bit.

 * If no bits are zero, returns @size.

 */

static inline

static __always_inline

unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)

{

	if (small_const_nbits(size)) {
@@ -402,7 +402,7 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) 
 *

 * Returns the bit number of the last set bit, or size.

 */

static inline

static __always_inline

unsigned long find_last_bit(const unsigned long *addr, unsigned long size)

{

	if (small_const_nbits(size)) {
@@ -425,7 +425,7 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) 
 * Returns the bit number for the next set bit, or first set bit up to @offset

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_and_bit_wrap(const unsigned long *addr1,

					const unsigned long *addr2,

					unsigned long size, unsigned long offset)
@@ -448,7 +448,7 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1, 
 * Returns the bit number for the next set bit, or first set bit up to @offset

 * If no bits are set, returns @size.

 */

static inline

static __always_inline

unsigned long find_next_bit_wrap(const unsigned long *addr,

					unsigned long size, unsigned long offset)

{
@@ -465,7 +465,7 @@ unsigned long find_next_bit_wrap(const unsigned long *addr, 
 * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing

 * before using it alone.

 */

static inline

static __always_inline

unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,

				 unsigned long start, unsigned long n)

{
@@ -506,20 +506,20 @@ extern unsigned long find_next_clump8(unsigned long *clump, 


#if defined(__LITTLE_ENDIAN)



static inline unsigned long find_next_zero_bit_le(const void *addr,

		unsigned long size, unsigned long offset)

static __always_inline

unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset)

{

	return find_next_zero_bit(addr, size, offset);

}



static inline unsigned long find_next_bit_le(const void *addr,

		unsigned long size, unsigned long offset)

static __always_inline

unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset)

{

	return find_next_bit(addr, size, offset);

}



static inline unsigned long find_first_zero_bit_le(const void *addr,

		unsigned long size)

static __always_inline

unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)

{

	return find_first_zero_bit(addr, size);

}
@@ -527,7 +527,7 @@ static inline unsigned long find_first_zero_bit_le(const void *addr, 
#elif defined(__BIG_ENDIAN)



#ifndef find_next_zero_bit_le

static inline

static __always_inline

unsigned long find_next_zero_bit_le(const void *addr, unsigned

		long size, unsigned long offset)

{
@@ -546,7 +546,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned 
#endif



#ifndef find_first_zero_bit_le

static inline

static __always_inline

unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)

{

	if (small_const_nbits(size)) {
@@ -560,7 +560,7 @@ unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) 
#endif



#ifndef find_next_bit_le

static inline

static __always_inline

unsigned long find_next_bit_le(const void *addr, unsigned

		long size, unsigned long offset)

{

include/linux/nodemask.h

+43 −43
Original line number Diff line number Diff line
@@ -107,11 +107,11 @@ extern nodemask_t _unused_nodemask_arg_; 
 */

#define nodemask_pr_args(maskp)	__nodemask_pr_numnodes(maskp), \

				__nodemask_pr_bits(maskp)

static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)

static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)

{

	return m ? MAX_NUMNODES : 0;

}

static inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)

static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)

{

	return m ? m->bits : NULL;

}
@@ -132,19 +132,19 @@ static __always_inline void __node_set(int node, volatile nodemask_t *dstp) 
}



#define node_clear(node, dst) __node_clear((node), &(dst))

static inline void __node_clear(int node, volatile nodemask_t *dstp)

static __always_inline void __node_clear(int node, volatile nodemask_t *dstp)

{

	clear_bit(node, dstp->bits);

}



#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)

static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)

static __always_inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)

{

	bitmap_fill(dstp->bits, nbits);

}



#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)

static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)

static __always_inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)

{

	bitmap_zero(dstp->bits, nbits);

}
@@ -154,14 +154,14 @@ static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) 


#define node_test_and_set(node, nodemask) \

			__node_test_and_set((node), &(nodemask))

static inline bool __node_test_and_set(int node, nodemask_t *addr)

static __always_inline bool __node_test_and_set(int node, nodemask_t *addr)

{

	return test_and_set_bit(node, addr->bits);

}



#define nodes_and(dst, src1, src2) \

			__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)

static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,

static __always_inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -169,7 +169,7 @@ static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p, 


#define nodes_or(dst, src1, src2) \

			__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)

static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,

static __always_inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -177,7 +177,7 @@ static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p, 


#define nodes_xor(dst, src1, src2) \

			__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)

static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,

static __always_inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -185,7 +185,7 @@ static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p, 


#define nodes_andnot(dst, src1, src2) \

			__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)

static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,

static __always_inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
@@ -193,7 +193,7 @@ static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p, 


#define nodes_complement(dst, src) \

			__nodes_complement(&(dst), &(src), MAX_NUMNODES)

static inline void __nodes_complement(nodemask_t *dstp,

static __always_inline void __nodes_complement(nodemask_t *dstp,

					const nodemask_t *srcp, unsigned int nbits)

{

	bitmap_complement(dstp->bits, srcp->bits, nbits);
@@ -201,7 +201,7 @@ static inline void __nodes_complement(nodemask_t *dstp, 


#define nodes_equal(src1, src2) \

			__nodes_equal(&(src1), &(src2), MAX_NUMNODES)

static inline bool __nodes_equal(const nodemask_t *src1p,

static __always_inline bool __nodes_equal(const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	return bitmap_equal(src1p->bits, src2p->bits, nbits);
@@ -209,7 +209,7 @@ static inline bool __nodes_equal(const nodemask_t *src1p, 


#define nodes_intersects(src1, src2) \

			__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)

static inline bool __nodes_intersects(const nodemask_t *src1p,

static __always_inline bool __nodes_intersects(const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
@@ -217,33 +217,33 @@ static inline bool __nodes_intersects(const nodemask_t *src1p, 


#define nodes_subset(src1, src2) \

			__nodes_subset(&(src1), &(src2), MAX_NUMNODES)

static inline bool __nodes_subset(const nodemask_t *src1p,

static __always_inline bool __nodes_subset(const nodemask_t *src1p,

					const nodemask_t *src2p, unsigned int nbits)

{

	return bitmap_subset(src1p->bits, src2p->bits, nbits);

}



#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)

static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)

static __always_inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)

{

	return bitmap_empty(srcp->bits, nbits);

}



#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)

static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)

static __always_inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)

{

	return bitmap_full(srcp->bits, nbits);

}



#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)

static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)

static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)

{

	return bitmap_weight(srcp->bits, nbits);

}



#define nodes_shift_right(dst, src, n) \

			__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)

static inline void __nodes_shift_right(nodemask_t *dstp,

static __always_inline void __nodes_shift_right(nodemask_t *dstp,

					const nodemask_t *srcp, int n, int nbits)

{

	bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
@@ -251,7 +251,7 @@ static inline void __nodes_shift_right(nodemask_t *dstp, 


#define nodes_shift_left(dst, src, n) \

			__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)

static inline void __nodes_shift_left(nodemask_t *dstp,

static __always_inline void __nodes_shift_left(nodemask_t *dstp,

					const nodemask_t *srcp, int n, int nbits)

{

	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
@@ -261,13 +261,13 @@ static inline void __nodes_shift_left(nodemask_t *dstp, 
          > MAX_NUMNODES, then the silly min_ts could be dropped. */



#define first_node(src) __first_node(&(src))

static inline unsigned int __first_node(const nodemask_t *srcp)

static __always_inline unsigned int __first_node(const nodemask_t *srcp)

{

	return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));

}



#define next_node(n, src) __next_node((n), &(src))

static inline unsigned int __next_node(int n, const nodemask_t *srcp)

static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp)

{

	return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));

}
@@ -277,7 +277,7 @@ static inline unsigned int __next_node(int n, const nodemask_t *srcp) 
 * the first node in src if needed.  Returns MAX_NUMNODES if src is empty.

 */

#define next_node_in(n, src) __next_node_in((n), &(src))

static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)

static __always_inline unsigned int __next_node_in(int node, const nodemask_t *srcp)

{

	unsigned int ret = __next_node(node, srcp);
@@ -286,7 +286,7 @@ static inline unsigned int __next_node_in(int node, const nodemask_t *srcp) 
	return ret;

}



static inline void init_nodemask_of_node(nodemask_t *mask, int node)

static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node)

{

	nodes_clear(*mask);

	node_set(node, *mask);
@@ -304,7 +304,7 @@ static inline void init_nodemask_of_node(nodemask_t *mask, int node) 
})



#define first_unset_node(mask) __first_unset_node(&(mask))

static inline unsigned int __first_unset_node(const nodemask_t *maskp)

static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp)

{

	return min_t(unsigned int, MAX_NUMNODES,

			find_first_zero_bit(maskp->bits, MAX_NUMNODES));
@@ -338,21 +338,21 @@ static inline unsigned int __first_unset_node(const nodemask_t *maskp) 


#define nodemask_parse_user(ubuf, ulen, dst) \

		__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)

static inline int __nodemask_parse_user(const char __user *buf, int len,

static __always_inline int __nodemask_parse_user(const char __user *buf, int len,

					nodemask_t *dstp, int nbits)

{

	return bitmap_parse_user(buf, len, dstp->bits, nbits);

}



#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)

static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)

static __always_inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)

{

	return bitmap_parselist(buf, dstp->bits, nbits);

}



#define node_remap(oldbit, old, new) \

		__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)

static inline int __node_remap(int oldbit,

static __always_inline int __node_remap(int oldbit,

		const nodemask_t *oldp, const nodemask_t *newp, int nbits)

{

	return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
@@ -360,7 +360,7 @@ static inline int __node_remap(int oldbit, 


#define nodes_remap(dst, src, old, new) \

		__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)

static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,

static __always_inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,

		const nodemask_t *oldp, const nodemask_t *newp, int nbits)

{

	bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
@@ -368,7 +368,7 @@ static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp, 


#define nodes_onto(dst, orig, relmap) \

		__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)

static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,

static __always_inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,

		const nodemask_t *relmapp, int nbits)

{

	bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
@@ -376,7 +376,7 @@ static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp, 


#define nodes_fold(dst, orig, sz) \

		__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)

static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,

static __always_inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,

		int sz, int nbits)

{

	bitmap_fold(dstp->bits, origp->bits, sz, nbits);
@@ -418,22 +418,22 @@ enum node_states { 
extern nodemask_t node_states[NR_NODE_STATES];



#if MAX_NUMNODES > 1

static inline int node_state(int node, enum node_states state)

static __always_inline int node_state(int node, enum node_states state)

{

	return node_isset(node, node_states[state]);

}



static inline void node_set_state(int node, enum node_states state)

static __always_inline void node_set_state(int node, enum node_states state)

{

	__node_set(node, &node_states[state]);

}



static inline void node_clear_state(int node, enum node_states state)

static __always_inline void node_clear_state(int node, enum node_states state)

{

	__node_clear(node, &node_states[state]);

}



static inline int num_node_state(enum node_states state)

static __always_inline int num_node_state(enum node_states state)

{

	return nodes_weight(node_states[state]);

}
@@ -443,11 +443,11 @@ static inline int num_node_state(enum node_states state) 


#define first_online_node	first_node(node_states[N_ONLINE])

#define first_memory_node	first_node(node_states[N_MEMORY])

static inline unsigned int next_online_node(int nid)

static __always_inline unsigned int next_online_node(int nid)

{

	return next_node(nid, node_states[N_ONLINE]);

}

static inline unsigned int next_memory_node(int nid)

static __always_inline unsigned int next_memory_node(int nid)

{

	return next_node(nid, node_states[N_MEMORY]);

}
@@ -455,13 +455,13 @@ static inline unsigned int next_memory_node(int nid) 
extern unsigned int nr_node_ids;

extern unsigned int nr_online_nodes;



static inline void node_set_online(int nid)

static __always_inline void node_set_online(int nid)

{

	node_set_state(nid, N_ONLINE);

	nr_online_nodes = num_node_state(N_ONLINE);

}



static inline void node_set_offline(int nid)

static __always_inline void node_set_offline(int nid)

{

	node_clear_state(nid, N_ONLINE);

	nr_online_nodes = num_node_state(N_ONLINE);
@@ -469,20 +469,20 @@ static inline void node_set_offline(int nid) 


#else



static inline int node_state(int node, enum node_states state)

static __always_inline int node_state(int node, enum node_states state)

{

	return node == 0;

}



static inline void node_set_state(int node, enum node_states state)

static __always_inline void node_set_state(int node, enum node_states state)

{

}



static inline void node_clear_state(int node, enum node_states state)

static __always_inline void node_clear_state(int node, enum node_states state)

{

}



static inline int num_node_state(enum node_states state)

static __always_inline int num_node_state(enum node_states state)

{

	return 1;

}
@@ -502,7 +502,7 @@ static inline int num_node_state(enum node_states state) 


#endif



static inline int node_random(const nodemask_t *maskp)

static __always_inline int node_random(const nodemask_t *maskp)

{

#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)

	int w, bit;