Commit 38ced43b authored by Kent Overstreet's avatar Kent Overstreet
Browse files

bcachefs: Inline btree write buffer sort 



The sort in the btree write buffer flush path is a very hot path, and
it's particularly performance sensitive since it's single threaded and
can block every other thread on a multithreaded write workload.

It's well worth doing a sort with inlined cmp and swap functions.

Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
parent 09caeabe

fs/bcachefs/btree_write_buffer.c

+82 −11
Original line number Diff line number Diff line
@@ -11,21 +11,95 @@ 
#include "journal_reclaim.h"



#include <linux/prefetch.h>

#include <linux/sort.h>



static int bch2_btree_write_buffer_journal_flush(struct journal *,

				struct journal_entry_pin *, u64);



static int bch2_journal_keys_to_write_buffer(struct bch_fs *, struct journal_buf *);



static inline int wb_key_cmp(const void *_l, const void *_r)

static inline bool __wb_key_cmp(const struct wb_key_ref *l, const struct wb_key_ref *r)

{

	return (cmp_int(l->hi, r->hi) ?:

		cmp_int(l->mi, r->mi) ?:

		cmp_int(l->lo, r->lo)) >= 0;

}



static inline bool wb_key_cmp(const struct wb_key_ref *l, const struct wb_key_ref *r)

{

#ifdef CONFIG_X86_64

	int cmp;



	asm("mov   (%[l]), %%rax;"

	    "sub   (%[r]), %%rax;"

	    "mov  8(%[l]), %%rax;"

	    "sbb  8(%[r]), %%rax;"

	    "mov 16(%[l]), %%rax;"

	    "sbb 16(%[r]), %%rax;"

	    : "=@ccae" (cmp)

	    : [l] "r" (l), [r] "r" (r)

	    : "rax", "cc");



	EBUG_ON(cmp != __wb_key_cmp(l, r));

	return cmp;

#else

	return __wb_key_cmp(l, r);

#endif

}



/* Compare excluding idx, the low 24 bits: */

static inline bool wb_key_eq(const void *_l, const void *_r)

{

	const struct wb_key_ref *l = _l;

	const struct wb_key_ref *r = _r;



	return  cmp_int(l->hi, r->hi) ?:

		cmp_int(l->mi, r->mi) ?:

		cmp_int(l->lo, r->lo);

	return !((l->hi ^ r->hi)|

		 (l->mi ^ r->mi)|

		 ((l->lo >> 24) ^ (r->lo >> 24)));

}



static noinline void wb_sort(struct wb_key_ref *base, size_t num)

{

	size_t n = num, a = num / 2;



	if (!a)		/* num < 2 || size == 0 */

		return;



	for (;;) {

		size_t b, c, d;



		if (a)			/* Building heap: sift down --a */

			--a;

		else if (--n)		/* Sorting: Extract root to --n */

			swap(base[0], base[n]);

		else			/* Sort complete */

			break;



		/*

		 * Sift element at "a" down into heap.  This is the

		 * "bottom-up" variant, which significantly reduces

		 * calls to cmp_func(): we find the sift-down path all

		 * the way to the leaves (one compare per level), then

		 * backtrack to find where to insert the target element.

		 *

		 * Because elements tend to sift down close to the leaves,

		 * this uses fewer compares than doing two per level

		 * on the way down.  (A bit more than half as many on

		 * average, 3/4 worst-case.)

		 */

		for (b = a; c = 2*b + 1, (d = c + 1) < n;)

			b = wb_key_cmp(base + c, base + d) ? c : d;

		if (d == n)		/* Special case last leaf with no sibling */

			b = c;



		/* Now backtrack from "b" to the correct location for "a" */

		while (b != a && wb_key_cmp(base + a, base + b))

			b = (b - 1) / 2;

		c = b;			/* Where "a" belongs */

		while (b != a) {	/* Shift it into place */

			b = (b - 1) / 2;

			swap(base[b], base[c]);

		}

	}

}



static noinline int wb_flush_one_slowpath(struct btree_trans *trans,
@@ -188,7 +262,7 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) 
	for (size_t i = 0; i < wb->flushing.keys.nr; i++) {

		wb->sorted.data[i].idx = i;

		wb->sorted.data[i].btree = wb->flushing.keys.data[i].btree;

		wb->sorted.data[i].pos = wb->flushing.keys.data[i].k.k.p;

		memcpy(&wb->sorted.data[i].pos, &wb->flushing.keys.data[i].k.k.p, sizeof(struct bpos));

	}

	wb->sorted.nr = wb->flushing.keys.nr;
@@ -206,9 +280,7 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) 
	 * If that happens, simply skip the key so we can optimistically insert

	 * as many keys as possible in the fast path.

	 */

	sort(wb->sorted.data, wb->sorted.nr,

	     sizeof(wb->sorted.data[0]),

	     wb_key_cmp, NULL);

	wb_sort(wb->sorted.data, wb->sorted.nr);



	darray_for_each(wb->sorted, i) {

		struct btree_write_buffered_key *k = &wb->flushing.keys.data[i->idx];
@@ -219,8 +291,7 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) 
		BUG_ON(!k->journal_seq);



		if (i + 1 < &darray_top(wb->sorted) &&

		    i[0].btree == i[1].btree &&

		    bpos_eq(i[0].pos, i[1].pos)) {

		    wb_key_eq(i, i + 1)) {

			struct btree_write_buffered_key *n = &wb->flushing.keys.data[i[1].idx];



			skipped++;

fs/bcachefs/btree_write_buffer_types.h

+2 −2
Original line number Diff line number Diff line
@@ -13,11 +13,11 @@ union { 
	struct {

#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__

		unsigned			idx:24;

		struct bpos			pos;

		u8				pos[sizeof(struct bpos)];

		enum btree_id			btree:8;

#else

		enum btree_id			btree:8;

		struct bpos			pos;

		u8				pos[sizeof(struct bpos)];

		unsigned			idx:24;

#endif

	} __packed;