Commit 62de6e16 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'sched-core-2025-01-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull scheduler updates from Ingo Molnar:
 "Fair scheduler (SCHED_FAIR) enhancements:

   - Behavioral improvements:
      - Untangle NEXT_BUDDY and pick_next_task() (Peter Zijlstra)

   - Delayed-dequeue enhancements & fixes: (Vincent Guittot)
      - Rename h_nr_running into h_nr_queued
      - Add new cfs_rq.h_nr_runnable
      - Use the new cfs_rq.h_nr_runnable
      - Removed unsued cfs_rq.h_nr_delayed
      - Rename cfs_rq.idle_h_nr_running into h_nr_idle
      - Remove unused cfs_rq.idle_nr_running
      - Rename cfs_rq.nr_running into nr_queued
      - Do not try to migrate delayed dequeue task
      - Fix variable declaration position
      - Encapsulate set custom slice in a __setparam_fair() function

   - Fixes:
      - Fix race between yield_to() and try_to_wake_up() (Tianchen Ding)
      - Fix CPU bandwidth limit bypass during CPU hotplug (Vishal
        Chourasia)

   - Cleanups:
      - Clean up in migrate_degrades_locality() to improve readability
        (Peter Zijlstra)
      - Mark m*_vruntime() with __maybe_unused (Andy Shevchenko)
      - Update comments after sched_tick() rename (Sebastian Andrzej
        Siewior)
      - Remove CONFIG_CFS_BANDWIDTH=n definition of cfs_bandwidth_used()
        (Valentin Schneider)

  Deadline scheduler (SCHED_DL) enhancements:

   - Restore dl_server bandwidth on non-destructive root domain changes
     (Juri Lelli)

   - Correctly account for allocated bandwidth during hotplug (Juri
     Lelli)

   - Check bandwidth overflow earlier for hotplug (Juri Lelli)

   - Clean up goto label in pick_earliest_pushable_dl_task() (John
     Stultz)

   - Consolidate timer cancellation (Wander Lairson Costa)

  Load-balancer enhancements:

   - Improve performance by prioritizing migrating eligible tasks in
     sched_balance_rq() (Hao Jia)

   - Do not compute NUMA Balancing stats unnecessarily during
     load-balancing (K Prateek Nayak)

   - Do not compute overloaded status unnecessarily during
     load-balancing (K Prateek Nayak)

  Generic scheduling code enhancements:

   - Use READ_ONCE() in task_on_rq_queued(), to consistently use the
     WRITE_ONCE() updated ->on_rq field (Harshit Agarwal)

  Isolated CPUs support enhancements: (Waiman Long)

   - Make "isolcpus=nohz" equivalent to "nohz_full"
   - Consolidate housekeeping cpumasks that are always identical
   - Remove HK_TYPE_SCHED
   - Unify HK_TYPE_{TIMER|TICK|MISC} to HK_TYPE_KERNEL_NOISE

  RSEQ enhancements:

   - Validate read-only fields under DEBUG_RSEQ config (Mathieu
     Desnoyers)

  PSI enhancements:

   - Fix race when task wakes up before psi_sched_switch() adjusts flags
     (Chengming Zhou)

  IRQ time accounting performance enhancements: (Yafang Shao)

   - Define sched_clock_irqtime as static key
   - Don't account irq time if sched_clock_irqtime is disabled

  Virtual machine scheduling enhancements:

   - Don't try to catch up excess steal time (Suleiman Souhlal)

  Heterogenous x86 CPU scheduling enhancements: (K Prateek Nayak)

   - Convert "sysctl_sched_itmt_enabled" to boolean
   - Use guard() for itmt_update_mutex
   - Move the "sched_itmt_enabled" sysctl to debugfs
   - Remove x86_smt_flags and use cpu_smt_flags directly
   - Use x86_sched_itmt_flags for PKG domain unconditionally

  Debugging code & instrumentation enhancements:

   - Change need_resched warnings to pr_err() (David Rientjes)
   - Print domain name in /proc/schedstat (K Prateek Nayak)
   - Fix value reported by hot tasks pulled in /proc/schedstat (Peter
     Zijlstra)
   - Report the different kinds of imbalances in /proc/schedstat
     (Swapnil Sapkal)
   - Move sched domain name out of CONFIG_SCHED_DEBUG (Swapnil Sapkal)
   - Update Schedstat version to 17 (Swapnil Sapkal)"

* tag 'sched-core-2025-01-21' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (48 commits)
  rseq: Fix rseq unregistration regression
  psi: Fix race when task wakes up before psi_sched_switch() adjusts flags
  sched, psi: Don't account irq time if sched_clock_irqtime is disabled
  sched: Don't account irq time if sched_clock_irqtime is disabled
  sched: Define sched_clock_irqtime as static key
  sched/fair: Do not compute overloaded status unnecessarily during lb
  sched/fair: Do not compute NUMA Balancing stats unnecessarily during lb
  x86/topology: Use x86_sched_itmt_flags for PKG domain unconditionally
  x86/topology: Remove x86_smt_flags and use cpu_smt_flags directly
  x86/itmt: Move the "sched_itmt_enabled" sysctl to debugfs
  x86/itmt: Use guard() for itmt_update_mutex
  x86/itmt: Convert "sysctl_sched_itmt_enabled" to boolean
  sched/core: Prioritize migrating eligible tasks in sched_balance_rq()
  sched/debug: Change need_resched warnings to pr_err
  sched/fair: Encapsulate set custom slice in a __setparam_fair() function
  sched: Fix race between yield_to() and try_to_wake_up()
  docs: Update Schedstat version to 17
  sched/stats: Print domain name in /proc/schedstat
  sched: Move sched domain name out of CONFIG_SCHED_DEBUG
  sched: Report the different kinds of imbalances in /proc/schedstat
  ...
parents 858df1de 40724eca

Documentation/admin-guide/kernel-parameters.txt

+3 −1
Original line number Diff line number Diff line
@@ -2506,7 +2506,9 @@ 
			specified in the flag list (default: domain):



			nohz

			  Disable the tick when a single task runs.

			  Disable the tick when a single task runs as well as

			  disabling other kernel noises like having RCU callbacks

			  offloaded. This is equivalent to the nohz_full parameter.



			  A residual 1Hz tick is offloaded to workqueues, which you

			  need to affine to housekeeping through the global

Documentation/scheduler/sched-stats.rst

+75 −51
Original line number Diff line number Diff line
@@ -2,6 +2,12 @@ 
Scheduler Statistics

====================



Version 17 of schedstats removed 'lb_imbalance' field as it has no

significance anymore and instead added more relevant fields namely

'lb_imbalance_load', 'lb_imbalance_util', 'lb_imbalance_task' and

'lb_imbalance_misfit'. The domain field prints the name of the

corresponding sched domain from this version onwards.



Version 16 of schedstats changed the order of definitions within

'enum cpu_idle_type', which changed the order of [CPU_MAX_IDLE_TYPES]

columns in show_schedstat(). In particular the position of CPU_IDLE
@@ -9,7 +15,9 @@ and __CPU_NOT_IDLE changed places. The size of the array is unchanged. 


Version 15 of schedstats dropped counters for some sched_yield:

yld_exp_empty, yld_act_empty and yld_both_empty. Otherwise, it is

identical to version 14.

identical to version 14. Details are available at



	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/scheduler/sched-stats.txt?id=1e1dbb259c79b



Version 14 of schedstats includes support for sched_domains, which hit the

mainline kernel in 2.6.20 although it is identical to the stats from version
@@ -26,7 +34,14 @@ cpus on the machine, while domain0 is the most tightly focused domain, 
sometimes balancing only between pairs of cpus.  At this time, there

are no architectures which need more than three domain levels. The first

field in the domain stats is a bit map indicating which cpus are affected

by that domain.

by that domain. Details are available at



	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/sched-stats.txt?id=b762f3ffb797c



The schedstat documentation is maintained version 10 onwards and is not

updated for version 11 and 12. The details for version 10 are available at



	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/sched-stats.txt?id=1da177e4c3f4



These fields are counters, and only increment.  Programs which make use

of these will need to start with a baseline observation and then calculate
@@ -71,88 +86,97 @@ Domain statistics 
-----------------

One of these is produced per domain for each cpu described. (Note that if

CONFIG_SMP is not defined, *no* domains are utilized and these lines

will not appear in the output.)

will not appear in the output. <name> is an extension to the domain field

that prints the name of the corresponding sched domain. It can appear in

schedstat version 17 and above, and requires CONFIG_SCHED_DEBUG.)



domain<N> <cpumask> 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36

domain<N> <name> <cpumask> 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45



The first field is a bit mask indicating what cpus this domain operates over.



The next 24 are a variety of sched_balance_rq() statistics in grouped into types

of idleness (idle, busy, and newly idle):

The next 33 are a variety of sched_balance_rq() statistics in grouped into types

of idleness (busy, idle and newly idle):



    1)  # of times in this domain sched_balance_rq() was called when the

        cpu was busy

    2)  # of times in this domain sched_balance_rq() checked but found the

        load did not require balancing when busy

    3)  # of times in this domain sched_balance_rq() tried to move one or

        more tasks and failed, when the cpu was busy

    4)  Total imbalance in load when the cpu was busy

    5)  Total imbalance in utilization when the cpu was busy

    6)  Total imbalance in number of tasks when the cpu was busy

    7)  Total imbalance due to misfit tasks when the cpu was busy

    8)  # of times in this domain pull_task() was called when busy

    9)  # of times in this domain pull_task() was called even though the

        target task was cache-hot when busy

    10) # of times in this domain sched_balance_rq() was called but did not

        find a busier queue while the cpu was busy

    11) # of times in this domain a busier queue was found while the cpu

        was busy but no busier group was found



    12) # of times in this domain sched_balance_rq() was called when the

        cpu was idle

    2)  # of times in this domain sched_balance_rq() checked but found

    13) # of times in this domain sched_balance_rq() checked but found

        the load did not require balancing when the cpu was idle

    3)  # of times in this domain sched_balance_rq() tried to move one or

    14) # of times in this domain sched_balance_rq() tried to move one or

        more tasks and failed, when the cpu was idle

    4)  sum of imbalances discovered (if any) with each call to

        sched_balance_rq() in this domain when the cpu was idle

    5)  # of times in this domain pull_task() was called when the cpu

    15) Total imbalance in load when the cpu was idle

    16) Total imbalance in utilization when the cpu was idle

    17) Total imbalance in number of tasks when the cpu was idle

    18) Total imbalance due to misfit tasks when the cpu was idle

    19) # of times in this domain pull_task() was called when the cpu

        was idle

    6)  # of times in this domain pull_task() was called even though

    20) # of times in this domain pull_task() was called even though

        the target task was cache-hot when idle

    7)  # of times in this domain sched_balance_rq() was called but did

    21) # of times in this domain sched_balance_rq() was called but did

        not find a busier queue while the cpu was idle

    8)  # of times in this domain a busier queue was found while the

    22) # of times in this domain a busier queue was found while the

        cpu was idle but no busier group was found

    9)  # of times in this domain sched_balance_rq() was called when the

        cpu was busy

    10) # of times in this domain sched_balance_rq() checked but found the

        load did not require balancing when busy

    11) # of times in this domain sched_balance_rq() tried to move one or

        more tasks and failed, when the cpu was busy

    12) sum of imbalances discovered (if any) with each call to

        sched_balance_rq() in this domain when the cpu was busy

    13) # of times in this domain pull_task() was called when busy

    14) # of times in this domain pull_task() was called even though the

        target task was cache-hot when busy

    15) # of times in this domain sched_balance_rq() was called but did not

        find a busier queue while the cpu was busy

    16) # of times in this domain a busier queue was found while the cpu

        was busy but no busier group was found



    17) # of times in this domain sched_balance_rq() was called when the

        cpu was just becoming idle

    18) # of times in this domain sched_balance_rq() checked but found the

    23) # of times in this domain sched_balance_rq() was called when the

        was just becoming idle

    24) # of times in this domain sched_balance_rq() checked but found the

        load did not require balancing when the cpu was just becoming idle

    19) # of times in this domain sched_balance_rq() tried to move one or more

    25) # of times in this domain sched_balance_rq() tried to move one or more

        tasks and failed, when the cpu was just becoming idle

    20) sum of imbalances discovered (if any) with each call to

        sched_balance_rq() in this domain when the cpu was just becoming idle

    21) # of times in this domain pull_task() was called when newly idle

    22) # of times in this domain pull_task() was called even though the

    26) Total imbalance in load when the cpu was just becoming idle

    27) Total imbalance in utilization when the cpu was just becoming idle

    28) Total imbalance in number of tasks when the cpu was just becoming idle

    29) Total imbalance due to misfit tasks when the cpu was just becoming idle

    30) # of times in this domain pull_task() was called when newly idle

    31) # of times in this domain pull_task() was called even though the

        target task was cache-hot when just becoming idle

    23) # of times in this domain sched_balance_rq() was called but did not

    32) # of times in this domain sched_balance_rq() was called but did not

        find a busier queue while the cpu was just becoming idle

    24) # of times in this domain a busier queue was found while the cpu

    33) # of times in this domain a busier queue was found while the cpu

        was just becoming idle but no busier group was found



   Next three are active_load_balance() statistics:



    25) # of times active_load_balance() was called

    26) # of times active_load_balance() tried to move a task and failed

    27) # of times active_load_balance() successfully moved a task

    34) # of times active_load_balance() was called

    35) # of times active_load_balance() tried to move a task and failed

    36) # of times active_load_balance() successfully moved a task



   Next three are sched_balance_exec() statistics:



    28) sbe_cnt is not used

    29) sbe_balanced is not used

    30) sbe_pushed is not used

    37) sbe_cnt is not used

    38) sbe_balanced is not used

    39) sbe_pushed is not used



   Next three are sched_balance_fork() statistics:



    31) sbf_cnt is not used

    32) sbf_balanced is not used

    33) sbf_pushed is not used

    40) sbf_cnt is not used

    41) sbf_balanced is not used

    42) sbf_pushed is not used



   Next three are try_to_wake_up() statistics:



    34) # of times in this domain try_to_wake_up() awoke a task that

    43) # of times in this domain try_to_wake_up() awoke a task that

        last ran on a different cpu in this domain

    35) # of times in this domain try_to_wake_up() moved a task to the

    44) # of times in this domain try_to_wake_up() moved a task to the

        waking cpu because it was cache-cold on its own cpu anyway

    36) # of times in this domain try_to_wake_up() started passive balancing

    45) # of times in this domain try_to_wake_up() started passive balancing



/proc/<pid>/schedstat

---------------------

arch/x86/include/asm/topology.h

+2 −2
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ extern bool x86_topology_update; 
#include <asm/percpu.h>



DECLARE_PER_CPU_READ_MOSTLY(int, sched_core_priority);

extern unsigned int __read_mostly sysctl_sched_itmt_enabled;

extern bool __read_mostly sysctl_sched_itmt_enabled;



/* Interface to set priority of a cpu */

void sched_set_itmt_core_prio(int prio, int core_cpu);
@@ -264,7 +264,7 @@ void sched_clear_itmt_support(void); 


#else /* CONFIG_SCHED_MC_PRIO */



#define sysctl_sched_itmt_enabled	0

#define sysctl_sched_itmt_enabled	false

static inline void sched_set_itmt_core_prio(int prio, int core_cpu)

{

}

arch/x86/kernel/itmt.c

+33 −48
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <linux/sched.h>

#include <linux/cpumask.h>

#include <linux/cpuset.h>

#include <linux/debugfs.h>

#include <linux/mutex.h>

#include <linux/sysctl.h>

#include <linux/nodemask.h>
@@ -34,49 +35,38 @@ static bool __read_mostly sched_itmt_capable; 
 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max

 * Technology 3.0.

 *

 * It can be set via /proc/sys/kernel/sched_itmt_enabled

 * It can be set via /sys/kernel/debug/x86/sched_itmt_enabled

 */

unsigned int __read_mostly sysctl_sched_itmt_enabled;

bool __read_mostly sysctl_sched_itmt_enabled;



static int sched_itmt_update_handler(const struct ctl_table *table, int write,

				     void *buffer, size_t *lenp, loff_t *ppos)

static ssize_t sched_itmt_enabled_write(struct file *filp,

					const char __user *ubuf,

					size_t cnt, loff_t *ppos)

{

	unsigned int old_sysctl;

	int ret;

	ssize_t result;

	bool orig;



	mutex_lock(&itmt_update_mutex);

	guard(mutex)(&itmt_update_mutex);



	if (!sched_itmt_capable) {

		mutex_unlock(&itmt_update_mutex);

		return -EINVAL;

	}



	old_sysctl = sysctl_sched_itmt_enabled;

	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

	orig = sysctl_sched_itmt_enabled;

	result = debugfs_write_file_bool(filp, ubuf, cnt, ppos);



	if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) {

	if (sysctl_sched_itmt_enabled != orig) {

		x86_topology_update = true;

		rebuild_sched_domains();

	}



	mutex_unlock(&itmt_update_mutex);



	return ret;

	return result;

}



static struct ctl_table itmt_kern_table[] = {

	{

		.procname	= "sched_itmt_enabled",

		.data		= &sysctl_sched_itmt_enabled,

		.maxlen		= sizeof(unsigned int),

		.mode		= 0644,

		.proc_handler	= sched_itmt_update_handler,

		.extra1		= SYSCTL_ZERO,

		.extra2		= SYSCTL_ONE,

	},

static const struct file_operations dfs_sched_itmt_fops = {

	.read =         debugfs_read_file_bool,

	.write =        sched_itmt_enabled_write,

	.open =         simple_open,

	.llseek =       default_llseek,

};



static struct ctl_table_header *itmt_sysctl_header;

static struct dentry *dfs_sched_itmt;



/**

 * sched_set_itmt_support() - Indicate platform supports ITMT
@@ -97,16 +87,18 @@ static struct ctl_table_header *itmt_sysctl_header; 
 */

int sched_set_itmt_support(void)

{

	mutex_lock(&itmt_update_mutex);

	guard(mutex)(&itmt_update_mutex);



	if (sched_itmt_capable) {

		mutex_unlock(&itmt_update_mutex);

	if (sched_itmt_capable)

		return 0;

	}



	itmt_sysctl_header = register_sysctl("kernel", itmt_kern_table);

	if (!itmt_sysctl_header) {

		mutex_unlock(&itmt_update_mutex);

	dfs_sched_itmt = debugfs_create_file_unsafe("sched_itmt_enabled",

						    0644,

						    arch_debugfs_dir,

						    &sysctl_sched_itmt_enabled,

						    &dfs_sched_itmt_fops);

	if (IS_ERR_OR_NULL(dfs_sched_itmt)) {

		dfs_sched_itmt = NULL;

		return -ENOMEM;

	}
@@ -117,8 +109,6 @@ int sched_set_itmt_support(void) 
	x86_topology_update = true;

	rebuild_sched_domains();



	mutex_unlock(&itmt_update_mutex);



	return 0;

}
@@ -134,18 +124,15 @@ int sched_set_itmt_support(void) 
 */

void sched_clear_itmt_support(void)

{

	mutex_lock(&itmt_update_mutex);

	guard(mutex)(&itmt_update_mutex);



	if (!sched_itmt_capable) {

		mutex_unlock(&itmt_update_mutex);

	if (!sched_itmt_capable)

		return;

	}



	sched_itmt_capable = false;



	if (itmt_sysctl_header) {

		unregister_sysctl_table(itmt_sysctl_header);

		itmt_sysctl_header = NULL;

	}

	debugfs_remove(dfs_sched_itmt);

	dfs_sched_itmt = NULL;



	if (sysctl_sched_itmt_enabled) {

		/* disable sched_itmt if we are no longer ITMT capable */
@@ -153,8 +140,6 @@ void sched_clear_itmt_support(void) 
		x86_topology_update = true;

		rebuild_sched_domains();

	}



	mutex_unlock(&itmt_update_mutex);

}



int arch_asym_cpu_priority(int cpu)

arch/x86/kernel/smpboot.c

+2 −17
Original line number Diff line number Diff line
@@ -483,12 +483,6 @@ static int x86_core_flags(void) 
	return cpu_core_flags() | x86_sched_itmt_flags();

}

#endif

#ifdef CONFIG_SCHED_SMT

static int x86_smt_flags(void)

{

	return cpu_smt_flags();

}

#endif

#ifdef CONFIG_SCHED_CLUSTER

static int x86_cluster_flags(void)

{
@@ -496,15 +490,6 @@ static int x86_cluster_flags(void) 
}

#endif



static int x86_die_flags(void)

{

	if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU) ||

	    cpu_feature_enabled(X86_FEATURE_AMD_HETEROGENEOUS_CORES))

		return x86_sched_itmt_flags();



	return 0;

}



/*

 * Set if a package/die has multiple NUMA nodes inside.

 * AMD Magny-Cours, Intel Cluster-on-Die, and Intel
@@ -520,7 +505,7 @@ static void __init build_sched_topology(void) 


#ifdef CONFIG_SCHED_SMT

	x86_topology[i++] = (struct sched_domain_topology_level){

		cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT)

		cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT)

	};

#endif

#ifdef CONFIG_SCHED_CLUSTER
@@ -540,7 +525,7 @@ static void __init build_sched_topology(void) 
	 */

	if (!x86_has_numa_in_package) {

		x86_topology[i++] = (struct sched_domain_topology_level){

			cpu_cpu_mask, x86_die_flags, SD_INIT_NAME(PKG)

			cpu_cpu_mask, x86_sched_itmt_flags, SD_INIT_NAME(PKG)

		};

	}