Commit af5b2619 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'wq-for-6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq 

Pull workqueue updates from Tejun Heo:

 - Prepare for defaulting to unbound workqueue. A separate branch was
   created to ease pulling in from other trees but none of the
   conversions have landed yet

 - Memory allocation profiling support added

 - Misc changes

* tag 'wq-for-6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq:
  workqueue: Use atomic_try_cmpxchg_relaxed() in tryinc_node_nr_active()
  workqueue: Remove unused work_on_cpu_safe
  workqueue: Add new WQ_PERCPU flag
  workqueue: Add system_percpu_wq and system_dfl_wq
  workqueue: Basic memory allocation profiling support
  workqueue: fix opencoded cpumask_next_and_wrap() in wq_select_unbound_cpu()
parents beace86e df316ab3

Documentation/core-api/workqueue.rst

+6 −0
Original line number Diff line number Diff line
@@ -183,6 +183,12 @@ resources, scheduled and executed. 
  BH work items cannot sleep. All other features such as delayed queueing,

  flushing and canceling are supported.



``WQ_PERCPU``

  Work items queued to a per-cpu wq are bound to a specific CPU.

  This flag is the right choice when cpu locality is important.



  This flag is the complement of ``WQ_UNBOUND``.



``WQ_UNBOUND``

  Work items queued to an unbound wq are served by the special

  worker-pools which host workers which are not bound to any

include/linux/workqueue.h

+14 −20
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#ifndef _LINUX_WORKQUEUE_H

#define _LINUX_WORKQUEUE_H



#include <linux/alloc_tag.h>

#include <linux/timer.h>

#include <linux/linkage.h>

#include <linux/bitops.h>
@@ -401,6 +402,7 @@ enum wq_flags { 
	 * http://thread.gmane.org/gmane.linux.kernel/1480396

	 */

	WQ_POWER_EFFICIENT	= 1 << 7,

	WQ_PERCPU		= 1 << 8, /* bound to a specific cpu */



	__WQ_DESTROYING		= 1 << 15, /* internal: workqueue is destroying */

	__WQ_DRAINING		= 1 << 16, /* internal: workqueue is draining */
@@ -427,7 +429,7 @@ enum wq_consts { 
/*

 * System-wide workqueues which are always present.

 *

 * system_wq is the one used by schedule[_delayed]_work[_on]().

 * system_percpu_wq is the one used by schedule[_delayed]_work[_on]().

 * Multi-CPU multi-threaded.  There are users which expect relatively

 * short queue flush time.  Don't queue works which can run for too

 * long.
@@ -438,7 +440,7 @@ enum wq_consts { 
 * system_long_wq is similar to system_wq but may host long running

 * works.  Queue flushing might take relatively long.

 *

 * system_unbound_wq is unbound workqueue.  Workers are not bound to

 * system_dfl_wq is unbound workqueue.  Workers are not bound to

 * any specific CPU, not concurrency managed, and all queued works are

 * executed immediately as long as max_active limit is not reached and

 * resources are available.
@@ -455,10 +457,12 @@ enum wq_consts { 
 * system_bh[_highpri]_wq are convenience interface to softirq. BH work items

 * are executed in the queueing CPU's BH context in the queueing order.

 */

extern struct workqueue_struct *system_wq;

extern struct workqueue_struct *system_wq; /* use system_percpu_wq, this will be removed */

extern struct workqueue_struct *system_percpu_wq;

extern struct workqueue_struct *system_highpri_wq;

extern struct workqueue_struct *system_long_wq;

extern struct workqueue_struct *system_unbound_wq;

extern struct workqueue_struct *system_dfl_wq;

extern struct workqueue_struct *system_freezable_wq;

extern struct workqueue_struct *system_power_efficient_wq;

extern struct workqueue_struct *system_freezable_power_efficient_wq;
@@ -505,7 +509,8 @@ void workqueue_softirq_dead(unsigned int cpu); 
 * Pointer to the allocated workqueue on success, %NULL on failure.

 */

__printf(1, 4) struct workqueue_struct *

alloc_workqueue(const char *fmt, unsigned int flags, int max_active, ...);

alloc_workqueue_noprof(const char *fmt, unsigned int flags, int max_active, ...);

#define alloc_workqueue(...)	alloc_hooks(alloc_workqueue_noprof(__VA_ARGS__))



#ifdef CONFIG_LOCKDEP

/**
@@ -544,8 +549,8 @@ alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags, int max_active, 
 * Pointer to the allocated workqueue on success, %NULL on failure.

 */

#define alloc_ordered_workqueue_lockdep_map(fmt, flags, lockdep_map, args...)	\

	alloc_workqueue_lockdep_map(fmt, WQ_UNBOUND | __WQ_ORDERED | (flags),	\

				    1, lockdep_map, ##args)

	alloc_hooks(alloc_workqueue_lockdep_map(fmt, WQ_UNBOUND | __WQ_ORDERED | (flags),\

						1, lockdep_map, ##args))

#endif



/**
@@ -577,7 +582,9 @@ alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags, int max_active, 


extern void destroy_workqueue(struct workqueue_struct *wq);



struct workqueue_attrs *alloc_workqueue_attrs(void);

struct workqueue_attrs *alloc_workqueue_attrs_noprof(void);

#define alloc_workqueue_attrs(...)	alloc_hooks(alloc_workqueue_attrs_noprof(__VA_ARGS__))



void free_workqueue_attrs(struct workqueue_attrs *attrs);

int apply_workqueue_attrs(struct workqueue_struct *wq,

			  const struct workqueue_attrs *attrs);
@@ -840,19 +847,6 @@ long work_on_cpu_key(int cpu, long (*fn)(void *), 
	work_on_cpu_key(_cpu, _fn, _arg, &__key);	\

})



long work_on_cpu_safe_key(int cpu, long (*fn)(void *),

			  void *arg, struct lock_class_key *key);



/*

 * A new key is defined for each caller to make sure the work

 * associated with the function doesn't share its locking class.

 */

#define work_on_cpu_safe(_cpu, _fn, _arg)		\

({							\

	static struct lock_class_key __key;		\

							\

	work_on_cpu_safe_key(_cpu, _fn, _arg, &__key);	\

})

#endif /* CONFIG_SMP */



#ifdef CONFIG_FREEZER

kernel/workqueue.c

+24 −50
Original line number Diff line number Diff line
@@ -505,12 +505,16 @@ static struct kthread_worker *pwq_release_worker __ro_after_init; 


struct workqueue_struct *system_wq __ro_after_init;

EXPORT_SYMBOL(system_wq);

struct workqueue_struct *system_percpu_wq __ro_after_init;

EXPORT_SYMBOL(system_percpu_wq);

struct workqueue_struct *system_highpri_wq __ro_after_init;

EXPORT_SYMBOL_GPL(system_highpri_wq);

struct workqueue_struct *system_long_wq __ro_after_init;

EXPORT_SYMBOL_GPL(system_long_wq);

struct workqueue_struct *system_unbound_wq __ro_after_init;

EXPORT_SYMBOL_GPL(system_unbound_wq);

struct workqueue_struct *system_dfl_wq __ro_after_init;

EXPORT_SYMBOL_GPL(system_dfl_wq);

struct workqueue_struct *system_freezable_wq __ro_after_init;

EXPORT_SYMBOL_GPL(system_freezable_wq);

struct workqueue_struct *system_power_efficient_wq __ro_after_init;
@@ -1686,18 +1690,15 @@ static void __pwq_activate_work(struct pool_workqueue *pwq, 
static bool tryinc_node_nr_active(struct wq_node_nr_active *nna)

{

	int max = READ_ONCE(nna->max);

	int old = atomic_read(&nna->nr);



	while (true) {

		int old, tmp;



		old = atomic_read(&nna->nr);

	do {

		if (old >= max)

			return false;

		tmp = atomic_cmpxchg_relaxed(&nna->nr, old, old + 1);

		if (tmp == old)

	} while (!atomic_try_cmpxchg_relaxed(&nna->nr, &old, old + 1));



	return true;

}

}



/**

 * pwq_tryinc_nr_active - Try to increment nr_active for a pwq
@@ -2221,12 +2222,9 @@ static int wq_select_unbound_cpu(int cpu) 
	}



	new_cpu = __this_cpu_read(wq_rr_cpu_last);

	new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);

	if (unlikely(new_cpu >= nr_cpu_ids)) {

		new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);

	new_cpu = cpumask_next_and_wrap(new_cpu, wq_unbound_cpumask, cpu_online_mask);

	if (unlikely(new_cpu >= nr_cpu_ids))

		return cpu;

	}

	__this_cpu_write(wq_rr_cpu_last, new_cpu);



	return new_cpu;
@@ -4629,7 +4627,7 @@ void free_workqueue_attrs(struct workqueue_attrs *attrs) 
 *

 * Return: The allocated new workqueue_attr on success. %NULL on failure.

 */

struct workqueue_attrs *alloc_workqueue_attrs(void)

struct workqueue_attrs *alloc_workqueue_attrs_noprof(void)

{

	struct workqueue_attrs *attrs;
@@ -5682,12 +5680,12 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, 
	else

		wq_size = sizeof(*wq);



	wq = kzalloc(wq_size, GFP_KERNEL);

	wq = kzalloc_noprof(wq_size, GFP_KERNEL);

	if (!wq)

		return NULL;



	if (flags & WQ_UNBOUND) {

		wq->unbound_attrs = alloc_workqueue_attrs();

		wq->unbound_attrs = alloc_workqueue_attrs_noprof();

		if (!wq->unbound_attrs)

			goto err_free_wq;

	}
@@ -5777,7 +5775,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, 
}



__printf(1, 4)

struct workqueue_struct *alloc_workqueue(const char *fmt,

struct workqueue_struct *alloc_workqueue_noprof(const char *fmt,

						unsigned int flags,

						int max_active, ...)

{
@@ -5794,7 +5792,7 @@ struct workqueue_struct *alloc_workqueue(const char *fmt, 


	return wq;

}

EXPORT_SYMBOL_GPL(alloc_workqueue);

EXPORT_SYMBOL_GPL(alloc_workqueue_noprof);



#ifdef CONFIG_LOCKDEP

__printf(1, 5)
@@ -6770,31 +6768,6 @@ long work_on_cpu_key(int cpu, long (*fn)(void *), 
	return wfc.ret;

}

EXPORT_SYMBOL_GPL(work_on_cpu_key);



/**

 * work_on_cpu_safe_key - run a function in thread context on a particular cpu

 * @cpu: the cpu to run on

 * @fn:  the function to run

 * @arg: the function argument

 * @key: The lock class key for lock debugging purposes

 *

 * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold

 * any locks which would prevent @fn from completing.

 *

 * Return: The value @fn returns.

 */

long work_on_cpu_safe_key(int cpu, long (*fn)(void *),

			  void *arg, struct lock_class_key *key)

{

	long ret = -ENODEV;



	cpus_read_lock();

	if (cpu_online(cpu))

		ret = work_on_cpu_key(cpu, fn, arg, key);

	cpus_read_unlock();

	return ret;

}

EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);

#endif /* CONFIG_SMP */



#ifdef CONFIG_FREEZER
@@ -7830,10 +7803,11 @@ void __init workqueue_init_early(void) 
	}



	system_wq = alloc_workqueue("events", 0, 0);

	system_percpu_wq = alloc_workqueue("events", 0, 0);

	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);

	system_long_wq = alloc_workqueue("events_long", 0, 0);

	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,

					    WQ_MAX_ACTIVE);

	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE);

	system_dfl_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE);

	system_freezable_wq = alloc_workqueue("events_freezable",

					      WQ_FREEZABLE, 0);

	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
@@ -7844,8 +7818,8 @@ void __init workqueue_init_early(void) 
	system_bh_wq = alloc_workqueue("events_bh", WQ_BH, 0);

	system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",

					       WQ_BH | WQ_HIGHPRI, 0);

	BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||

	       !system_unbound_wq || !system_freezable_wq ||

	BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq ||

	       !system_unbound_wq || !system_freezable_wq || !system_dfl_wq ||

	       !system_power_efficient_wq ||

	       !system_freezable_power_efficient_wq ||

	       !system_bh_wq || !system_bh_highpri_wq);