Commit d2ed0f20 authored by Kent Overstreet's avatar Kent Overstreet
Browse files

bcachefs: rcu_pending now works in userspace 



Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
parent 8e973a4f

fs/bcachefs/rcu_pending.c

+51 −4
Original line number Diff line number Diff line
@@ -115,24 +115,40 @@ static bool __rcu_pending_has_pending(struct rcu_pending_pcpu *p) 
static void rcu_pending_list_merge(struct rcu_pending_list *l1,

				   struct rcu_pending_list *l2)

{

#ifdef __KERNEL__

	if (!l1->head)

		l1->head = l2->head;

	else

		l1->tail->next = l2->head;

	l1->tail = l2->tail;

#else

	if (!l1->head)

		l1->head = l2->head;

	else

		l1->tail->next.next = (void *) l2->head;

#endif



	l1->tail = l2->tail;

	l2->head = l2->tail = NULL;

}



static void rcu_pending_list_add(struct rcu_pending_list *l,

				 struct rcu_head *n)

{

#ifdef __KERNEL__

	if (!l->head)

		l->head = n;

	else

		l->tail->next = n;

	l->tail = n;

	n->next = NULL;

#else

	if (!l->head)

		l->head = n;

	else

		l->tail->next.next = (void *) n;

	l->tail = n;

	n->next.next = NULL;

#endif

}



static void merge_expired_lists(struct rcu_pending_pcpu *p)
@@ -144,6 +160,19 @@ static void merge_expired_lists(struct rcu_pending_pcpu *p) 
			rcu_pending_list_merge(expired, i);

}



#ifndef __KERNEL__

static inline void kfree_bulk(size_t nr, void ** p)

{

	while (nr--)

		kfree(*p);

}



#define local_irq_save(flags)		\

do {					\

	flags = 0;			\

} while (0)

#endif



static noinline void __process_finished_items(struct rcu_pending *pending,

					      struct rcu_pending_pcpu *p,

					      unsigned long flags)
@@ -177,7 +206,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending, 


		while (list) {

			struct rcu_head *obj = list;

#ifdef __KERNEL__

			list = obj->next;

#else

			list = (void *) obj->next.next;

#endif



			/*

			 * low bit of pointer indicates whether rcu_head needs
@@ -204,7 +237,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending, 


		while (list) {

			struct rcu_head *obj = list;

#ifdef __KERNEL__

			list = obj->next;

#else

			list = (void *) obj->next.next;

#endif

			obj->func(obj);

		}

		break;
@@ -216,7 +253,11 @@ static noinline void __process_finished_items(struct rcu_pending *pending, 


		while (list) {

			struct rcu_head *obj = list;

#ifdef __KERNEL__

			list = obj->next;

#else

			list = (void *) obj->next.next;

#endif

			pending->process(pending, obj);

		}

		break;
@@ -265,12 +306,14 @@ static void rcu_pending_rcu_cb(struct rcu_head *rcu) 


	unsigned long flags;

	spin_lock_irqsave(&p->lock, flags);

	if (__rcu_pending_has_pending(p))

	if (__rcu_pending_has_pending(p)) {

		spin_unlock_irqrestore(&p->lock, flags);

		__call_rcu(p->parent->srcu, &p->cb, rcu_pending_rcu_cb);

	else

	} else {

		p->cb_armed = false;

		spin_unlock_irqrestore(&p->lock, flags);

	}

}



static __always_inline struct rcu_pending_seq *

get_object_radix(struct rcu_pending_pcpu *p, unsigned long seq)
@@ -489,7 +532,11 @@ static struct rcu_head *rcu_pending_pcpu_dequeue(struct rcu_pending_pcpu *p) 
	static_array_for_each(p->lists, i)

		if (i->head) {

			ret = i->head;

#ifdef __KERNEL__

			i->head = ret->next;

#else

			i->head = (void *) ret->next.next;

#endif

			if (!i->head)

				i->tail = NULL;

			goto out;

fs/bcachefs/rcu_pending.h

+2 −0
Original line number Diff line number Diff line
@@ -2,6 +2,8 @@ 
#ifndef _LINUX_RCU_PENDING_H

#define _LINUX_RCU_PENDING_H



#include <linux/rcupdate.h>



struct rcu_pending;

typedef void (*rcu_pending_process_fn)(struct rcu_pending *, struct rcu_head *);