Commit e2a33a2a authored by Kent Overstreet's avatar Kent Overstreet Committed by Andrew Morton
Browse files

lib/sort.c: add _nonatomic() variants with cond_resched() 

bcachefs calls sort() during recovery to sort all keys it found in the
journal, and this may be very large - gigabytes on large machines.

This has been causing "task blocked" warnings, so needs a
cond_resched().

[kent.overstreet@linux.dev: fix kerneldoc]
  Link: https://lkml.kernel.org/r/cgsr5a447pxqomc4gvznsp5yroqmif4omd7o5lsr2swifjhoic@yzjjrx2bvrq7
Link: https://lkml.kernel.org/r/20250326152606.2594920-1-kent.overstreet@linux.dev


Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
Cc: Kuan-Wei Chiu <visitorckw@gmail.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent c8b6d5dd

include/linux/sort.h

+11 −0
Original line number Diff line number Diff line
@@ -13,4 +13,15 @@ void sort(void *base, size_t num, size_t size, 
	  cmp_func_t cmp_func,

	  swap_func_t swap_func);



/* Versions that periodically call cond_resched(): */



void sort_r_nonatomic(void *base, size_t num, size_t size,

		      cmp_r_func_t cmp_func,

		      swap_r_func_t swap_func,

		      const void *priv);



void sort_nonatomic(void *base, size_t num, size_t size,

		    cmp_func_t cmp_func,

		    swap_func_t swap_func);



#endif

lib/sort.c

+79 −31
Original line number Diff line number Diff line
@@ -186,36 +186,13 @@ static size_t parent(size_t i, unsigned int lsbit, size_t size) 
	return i / 2;

}



/**

 * sort_r - sort an array of elements

 * @base: pointer to data to sort

 * @num: number of elements

 * @size: size of each element

 * @cmp_func: pointer to comparison function

 * @swap_func: pointer to swap function or NULL

 * @priv: third argument passed to comparison function

 *

 * This function does a heapsort on the given array.  You may provide

 * a swap_func function if you need to do something more than a memory

 * copy (e.g. fix up pointers or auxiliary data), but the built-in swap

 * avoids a slow retpoline and so is significantly faster.

 *

 * The comparison function must adhere to specific mathematical

 * properties to ensure correct and stable sorting:

 * - Antisymmetry: cmp_func(a, b) must return the opposite sign of

 * cmp_func(b, a).

 * - Transitivity: if cmp_func(a, b) <= 0 and cmp_func(b, c) <= 0, then

 * cmp_func(a, c) <= 0.

 *

 * Sorting time is O(n log n) both on average and worst-case. While

 * quicksort is slightly faster on average, it suffers from exploitable

 * O(n*n) worst-case behavior and extra memory requirements that make

 * it less suitable for kernel use.

 */

void sort_r(void *base, size_t num, size_t size,

#include <linux/sched.h>



static void __sort_r(void *base, size_t num, size_t size,

		     cmp_r_func_t cmp_func,

		     swap_r_func_t swap_func,

	    const void *priv)

		     const void *priv,

		     bool may_schedule)

{

	/* pre-scale counters for performance */

	size_t n = num * size, a = (num/2) * size;
@@ -286,6 +263,9 @@ void sort_r(void *base, size_t num, size_t size, 
			b = parent(b, lsbit, size);

			do_swap(base + b, base + c, size, swap_func, priv);

		}



		if (may_schedule)

			cond_resched();

	}



	n -= size;
@@ -293,8 +273,63 @@ void sort_r(void *base, size_t num, size_t size, 
	if (n == size * 2 && do_cmp(base, base + size, cmp_func, priv) > 0)

		do_swap(base, base + size, size, swap_func, priv);

}



/**

 * sort_r - sort an array of elements

 * @base: pointer to data to sort

 * @num: number of elements

 * @size: size of each element

 * @cmp_func: pointer to comparison function

 * @swap_func: pointer to swap function or NULL

 * @priv: third argument passed to comparison function

 *

 * This function does a heapsort on the given array.  You may provide

 * a swap_func function if you need to do something more than a memory

 * copy (e.g. fix up pointers or auxiliary data), but the built-in swap

 * avoids a slow retpoline and so is significantly faster.

 *

 * The comparison function must adhere to specific mathematical

 * properties to ensure correct and stable sorting:

 * - Antisymmetry: cmp_func(a, b) must return the opposite sign of

 * cmp_func(b, a).

 * - Transitivity: if cmp_func(a, b) <= 0 and cmp_func(b, c) <= 0, then

 * cmp_func(a, c) <= 0.

 *

 * Sorting time is O(n log n) both on average and worst-case. While

 * quicksort is slightly faster on average, it suffers from exploitable

 * O(n*n) worst-case behavior and extra memory requirements that make

 * it less suitable for kernel use.

 */

void sort_r(void *base, size_t num, size_t size,

	    cmp_r_func_t cmp_func,

	    swap_r_func_t swap_func,

	    const void *priv)

{

	__sort_r(base, num, size, cmp_func, swap_func, priv, false);

}

EXPORT_SYMBOL(sort_r);



/**

 * sort_r_nonatomic - sort an array of elements, with cond_resched

 * @base: pointer to data to sort

 * @num: number of elements

 * @size: size of each element

 * @cmp_func: pointer to comparison function

 * @swap_func: pointer to swap function or NULL

 * @priv: third argument passed to comparison function

 *

 * Same as sort_r, but preferred for larger arrays as it does a periodic

 * cond_resched().

 */

void sort_r_nonatomic(void *base, size_t num, size_t size,

		      cmp_r_func_t cmp_func,

		      swap_r_func_t swap_func,

		      const void *priv)

{

	__sort_r(base, num, size, cmp_func, swap_func, priv, true);

}

EXPORT_SYMBOL(sort_r_nonatomic);



void sort(void *base, size_t num, size_t size,

	  cmp_func_t cmp_func,

	  swap_func_t swap_func)
@@ -304,6 +339,19 @@ void sort(void *base, size_t num, size_t size, 
		.swap = swap_func,

	};



	return sort_r(base, num, size, _CMP_WRAPPER, SWAP_WRAPPER, &w);

	return __sort_r(base, num, size, _CMP_WRAPPER, SWAP_WRAPPER, &w, false);

}

EXPORT_SYMBOL(sort);



void sort_nonatomic(void *base, size_t num, size_t size,

		    cmp_func_t cmp_func,

		    swap_func_t swap_func)

{

	struct wrapper w = {

		.cmp  = cmp_func,

		.swap = swap_func,

	};



	return __sort_r(base, num, size, _CMP_WRAPPER, SWAP_WRAPPER, &w, true);

}

EXPORT_SYMBOL(sort_nonatomic);