Commit 9f66cff2 authored by Tejun Heo's avatar Tejun Heo
Browse files

workqueue: RCU protect wq->dfl_pwq and implement accessors for it 



wq->cpu_pwq is RCU protected but wq->dfl_pwq isn't. This is okay because
currently wq->dfl_pwq is used only accessed to install it into wq->cpu_pwq
which doesn't require RCU access. However, we want to be able to access
wq->dfl_pwq under RCU in the future to access its __pod_cpumask and the code
can be made easier to read by making the two pwq fields behave in the same
way.

- Make wq->dfl_pwq RCU protected.

- Add unbound_pwq_slot() and unbound_pwq() which can access both ->dfl_pwq
  and ->cpu_pwq. The former returns the double pointer that can be used
  access and update the pwqs. The latter performs locking check and
  dereferences the double pointer.

- pwq accesses and updates are converted to use unbound_pwq[_slot]().

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Reviewed-by: default avatarLai Jiangshan <jiangshanlai@gmail.com>
parent c5404d4e

kernel/workqueue.c

+40 −24
Original line number Diff line number Diff line
@@ -308,7 +308,7 @@ struct workqueue_struct { 
	int			saved_max_active; /* WQ: saved max_active */



	struct workqueue_attrs	*unbound_attrs;	/* PW: only for unbound wqs */

	struct pool_workqueue	*dfl_pwq;	/* PW: only for unbound wqs */

	struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */



#ifdef CONFIG_SYSFS

	struct wq_device	*wq_dev;	/* I: for sysfs interface */
@@ -639,6 +639,23 @@ static int worker_pool_assign_id(struct worker_pool *pool) 
	return ret;

}



static struct pool_workqueue __rcu **

unbound_pwq_slot(struct workqueue_struct *wq, int cpu)

{

       if (cpu >= 0)

               return per_cpu_ptr(wq->cpu_pwq, cpu);

       else

               return &wq->dfl_pwq;

}



/* @cpu < 0 for dfl_pwq */

static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)

{

	return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),

				     lockdep_is_held(&wq_pool_mutex) ||

				     lockdep_is_held(&wq->mutex));

}



static unsigned int work_color_to_flags(int color)

{

	return color << WORK_STRUCT_COLOR_SHIFT;
@@ -4328,10 +4345,11 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu, 
				"possible intersect\n");

}



/* install @pwq into @wq's cpu_pwq and return the old pwq */

/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */

static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,

					int cpu, struct pool_workqueue *pwq)

{

	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);

	struct pool_workqueue *old_pwq;



	lockdep_assert_held(&wq_pool_mutex);
@@ -4340,8 +4358,8 @@ static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, 
	/* link_pwq() can handle duplicate calls */

	link_pwq(pwq);



	old_pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));

	rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);

	old_pwq = rcu_access_pointer(*slot);

	rcu_assign_pointer(*slot, pwq);

	return old_pwq;

}
@@ -4441,14 +4459,11 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) 


	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);



	/* save the previous pwq and install the new one */

	/* save the previous pwqs and install the new ones */

	for_each_possible_cpu(cpu)

		ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,

							ctx->pwq_tbl[cpu]);



	/* @dfl_pwq might not have been used, ensure it's linked */

	link_pwq(ctx->dfl_pwq);

	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);

	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);



	mutex_unlock(&ctx->wq->mutex);

}
@@ -4558,9 +4573,7 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu, 


	/* nothing to do if the target cpumask matches the current pwq */

	wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);

	pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu),

					lockdep_is_held(&wq_pool_mutex));

	if (wqattrs_equal(target_attrs, pwq->pool->attrs))

	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))

		return;



	/* create a new pwq */
@@ -4578,10 +4591,11 @@ static void wq_update_pod(struct workqueue_struct *wq, int cpu, 


use_dfl_pwq:

	mutex_lock(&wq->mutex);

	raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);

	get_pwq(wq->dfl_pwq);

	raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);

	old_pwq = install_unbound_pwq(wq, cpu, wq->dfl_pwq);

	pwq = unbound_pwq(wq, -1);

	raw_spin_lock_irq(&pwq->pool->lock);

	get_pwq(pwq);

	raw_spin_unlock_irq(&pwq->pool->lock);

	old_pwq = install_unbound_pwq(wq, cpu, pwq);

out_unlock:

	mutex_unlock(&wq->mutex);

	put_pwq_unlocked(old_pwq);
@@ -4619,10 +4633,13 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) 


	cpus_read_lock();

	if (wq->flags & __WQ_ORDERED) {

		struct pool_workqueue *dfl_pwq;



		ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);

		/* there should only be single pwq for ordering guarantee */

		WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||

			      wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),

		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);

		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||

			      wq->pwqs.prev != &dfl_pwq->pwqs_node),

		     "ordering guarantee broken for workqueue %s\n", wq->name);

	} else {

		ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
@@ -4856,7 +4873,7 @@ static bool pwq_busy(struct pool_workqueue *pwq) 
		if (pwq->nr_in_flight[i])

			return true;



	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))

	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))

		return true;

	if (!pwq_is_empty(pwq))

		return true;
@@ -4940,13 +4957,12 @@ void destroy_workqueue(struct workqueue_struct *wq) 
	rcu_read_lock();



	for_each_possible_cpu(cpu) {

		pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));

		RCU_INIT_POINTER(*per_cpu_ptr(wq->cpu_pwq, cpu), NULL);

		put_pwq_unlocked(pwq);

		put_pwq_unlocked(unbound_pwq(wq, cpu));

		RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);

	}



	put_pwq_unlocked(wq->dfl_pwq);

	wq->dfl_pwq = NULL;

	put_pwq_unlocked(unbound_pwq(wq, -1));

	RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);



	rcu_read_unlock();

}