Commit 68573b93 authored by Uros Bizjak's avatar Uros Bizjak Committed by Kent Overstreet
Browse files

bcachefs: Use try_cmpxchg() family of functions instead of cmpxchg() 



Use try_cmpxchg() family of functions instead of
cmpxchg (*ptr, old, new) == old. x86 CMPXCHG instruction returns
success in ZF flag, so this change saves a compare after cmpxchg
(and related move instruction in front of cmpxchg).

Also, try_cmpxchg() implicitly assigns old *ptr value to "old" when
cmpxchg fails. There is no need to re-read the value in the loop.

No functional change intended.

Signed-off-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Brian Foster <bfoster@redhat.com>
Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
parent e76a2b65

fs/bcachefs/btree_cache.c

+4 −4
Original line number Diff line number Diff line
@@ -602,8 +602,8 @@ int bch2_btree_cache_cannibalize_lock(struct btree_trans *trans, struct closure 
	struct btree_cache *bc = &c->btree_cache;

	struct task_struct *old;



	old = cmpxchg(&bc->alloc_lock, NULL, current);

	if (old == NULL || old == current)

	old = NULL;

	if (try_cmpxchg(&bc->alloc_lock, &old, current) || old == current)

		goto success;



	if (!cl) {
@@ -614,8 +614,8 @@ int bch2_btree_cache_cannibalize_lock(struct btree_trans *trans, struct closure 
	closure_wait(&bc->alloc_wait, cl);



	/* Try again, after adding ourselves to waitlist */

	old = cmpxchg(&bc->alloc_lock, NULL, current);

	if (old == NULL || old == current) {

	old = NULL;

	if (try_cmpxchg(&bc->alloc_lock, &old, current) || old == current) {

		/* We raced */

		closure_wake_up(&bc->alloc_wait);

		goto success;

fs/bcachefs/btree_io.c

+11 −9
Original line number Diff line number Diff line
@@ -1796,15 +1796,16 @@ int bch2_btree_root_read(struct bch_fs *c, enum btree_id id, 
static void bch2_btree_complete_write(struct bch_fs *c, struct btree *b,

				      struct btree_write *w)

{

	unsigned long old, new, v = READ_ONCE(b->will_make_reachable);

	unsigned long old, new;



	old = READ_ONCE(b->will_make_reachable);

	do {

		old = new = v;

		new = old;

		if (!(old & 1))

			break;



		new &= ~1UL;

	} while ((v = cmpxchg(&b->will_make_reachable, old, new)) != old);

	} while (!try_cmpxchg(&b->will_make_reachable, &old, new));



	if (old & 1)

		closure_put(&((struct btree_update *) new)->cl);
@@ -1815,14 +1816,14 @@ static void bch2_btree_complete_write(struct bch_fs *c, struct btree *b, 
static void __btree_node_write_done(struct bch_fs *c, struct btree *b)

{

	struct btree_write *w = btree_prev_write(b);

	unsigned long old, new, v;

	unsigned long old, new;

	unsigned type = 0;



	bch2_btree_complete_write(c, b, w);



	v = READ_ONCE(b->flags);

	old = READ_ONCE(b->flags);

	do {

		old = new = v;

		new = old;



		if ((old & (1U << BTREE_NODE_dirty)) &&

		    (old & (1U << BTREE_NODE_need_write)) &&
@@ -1842,7 +1843,7 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b) 
			new &= ~(1U << BTREE_NODE_write_in_flight);

			new &= ~(1U << BTREE_NODE_write_in_flight_inner);

		}

	} while ((v = cmpxchg(&b->flags, old, new)) != old);

	} while (!try_cmpxchg(&b->flags, &old, new));



	if (new & (1U << BTREE_NODE_write_in_flight))

		__bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|type);
@@ -2014,8 +2015,9 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) 
	 * dirty bit requires a write lock, we can't race with other threads

	 * redirtying it:

	 */

	old = READ_ONCE(b->flags);

	do {

		old = new = READ_ONCE(b->flags);

		new = old;



		if (!(old & (1 << BTREE_NODE_dirty)))

			return;
@@ -2046,7 +2048,7 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) 
		new |=  (1 << BTREE_NODE_write_in_flight_inner);

		new |=  (1 << BTREE_NODE_just_written);

		new ^=  (1 << BTREE_NODE_write_idx);

	} while (cmpxchg_acquire(&b->flags, old, new) != old);

	} while (!try_cmpxchg_acquire(&b->flags, &old, new));



	if (new & (1U << BTREE_NODE_need_write))

		return;

fs/bcachefs/btree_trans_commit.c

+4 −4
Original line number Diff line number Diff line
@@ -228,14 +228,14 @@ static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, 
	struct btree_write *w = container_of(pin, struct btree_write, journal);

	struct btree *b = container_of(w, struct btree, writes[i]);

	struct btree_trans *trans = bch2_trans_get(c);

	unsigned long old, new, v;

	unsigned long old, new;

	unsigned idx = w - b->writes;



	btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read);

	v = READ_ONCE(b->flags);



	old = READ_ONCE(b->flags);

	do {

		old = new = v;

		new = old;



		if (!(old & (1 << BTREE_NODE_dirty)) ||

		    !!(old & (1 << BTREE_NODE_write_idx)) != idx ||
@@ -245,7 +245,7 @@ static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, 
		new &= ~BTREE_WRITE_TYPE_MASK;

		new |= BTREE_WRITE_journal_reclaim;

		new |= 1 << BTREE_NODE_need_write;

	} while ((v = cmpxchg(&b->flags, old, new)) != old);

	} while (!try_cmpxchg(&b->flags, &old, new));



	btree_node_write_if_need(c, b, SIX_LOCK_read);

	six_unlock_read(&b->c.lock);

fs/bcachefs/btree_update_interior.c

+4 −4
Original line number Diff line number Diff line
@@ -1356,7 +1356,7 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, 
	struct bch_fs *c = as->c;

	struct bkey_packed *k;

	struct printbuf buf = PRINTBUF;

	unsigned long old, new, v;

	unsigned long old, new;



	BUG_ON(insert->k.type == KEY_TYPE_btree_ptr_v2 &&

	       !btree_ptr_sectors_written(insert));
@@ -1395,14 +1395,14 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, 
	bch2_btree_bset_insert_key(trans, path, b, node_iter, insert);

	set_btree_node_dirty_acct(c, b);



	v = READ_ONCE(b->flags);

	old = READ_ONCE(b->flags);

	do {

		old = new = v;

		new = old;



		new &= ~BTREE_WRITE_TYPE_MASK;

		new |= BTREE_WRITE_interior;

		new |= 1 << BTREE_NODE_need_write;

	} while ((v = cmpxchg(&b->flags, old, new)) != old);

	} while (!try_cmpxchg(&b->flags, &old, new));



	printbuf_exit(&buf);

}

fs/bcachefs/buckets.c

+8 −9
Original line number Diff line number Diff line
@@ -916,13 +916,13 @@ void bch2_trans_account_disk_usage_change(struct btree_trans *trans) 
	 */

	s64 should_not_have_added = added - (s64) disk_res_sectors;

	if (unlikely(should_not_have_added > 0)) {

		u64 old, new, v = atomic64_read(&c->sectors_available);

		u64 old, new;



		old = atomic64_read(&c->sectors_available);

		do {

			old = v;

			new = max_t(s64, 0, old - should_not_have_added);

		} while ((v = atomic64_cmpxchg(&c->sectors_available,

					       old, new)) != old);

		} while (!atomic64_try_cmpxchg(&c->sectors_available,

					       &old, new));



		added -= should_not_have_added;

		warn = true;
@@ -1523,7 +1523,7 @@ int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, 
			      u64 sectors, int flags)

{

	struct bch_fs_pcpu *pcpu;

	u64 old, v, get;

	u64 old, get;

	s64 sectors_available;

	int ret;
@@ -1534,17 +1534,16 @@ int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, 
	if (sectors <= pcpu->sectors_available)

		goto out;



	v = atomic64_read(&c->sectors_available);

	old = atomic64_read(&c->sectors_available);

	do {

		old = v;

		get = min((u64) sectors + SECTORS_CACHE, old);



		if (get < sectors) {

			preempt_enable();

			goto recalculate;

		}

	} while ((v = atomic64_cmpxchg(&c->sectors_available,

				       old, old - get)) != old);

	} while (!atomic64_try_cmpxchg(&c->sectors_available,

				       &old, old - get));



	pcpu->sectors_available		+= get;