Commit a9af76a7 authored by Feng Tang's avatar Feng Tang Committed by Andrew Morton
Browse files

watchdog: add sys_info sysctls to dump sys info on system lockup 

When soft/hard lockup happens, developers may need different kinds of
system information (call-stacks, memory info, locks, etc.) to help
debugging.

Add 'softlockup_sys_info' and 'hardlockup_sys_info' sysctl knobs to take
human readable string like "tasks,mem,timers,locks,ftrace,...", and when
system lockup happens, all requested information will be printed out. 
(refer kernel/sys_info.c for more details).

Link: https://lkml.kernel.org/r/20251113111039.22701-4-feng.tang@linux.alibaba.com


Signed-off-by: default avatarFeng Tang <feng.tang@linux.alibaba.com>
Reviewed-by: default avatarPetr Mladek <pmladek@suse.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lance Yang <ioworker0@gmail.com>
Cc: "Paul E . McKenney" <paulmck@kernel.org>
Cc: Petr Mladek <pmladek@suse.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 8b2b9b4f

Documentation/admin-guide/sysctl/kernel.rst

+5 −0
Original line number Diff line number Diff line
@@ -582,6 +582,11 @@ if leaking kernel pointer values to unprivileged users is a concern. 
When ``kptr_restrict`` is set to 2, kernel pointers printed using

%pK will be replaced with 0s regardless of privileges.



softlockup_sys_info & hardlockup_sys_info

=========================================

A comma separated list of extra system information to be dumped when

soft/hard lockup is detected, for example, "tasks,mem,timers,locks,...".

Refer 'panic_sys_info' section below for more details.



modprobe

========

kernel/watchdog.c

+41 −3
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <linux/stop_machine.h>

#include <linux/sysctl.h>

#include <linux/tick.h>

#include <linux/sys_info.h>



#include <linux/sched/clock.h>

#include <linux/sched/debug.h>
@@ -65,6 +66,13 @@ int __read_mostly sysctl_hardlockup_all_cpu_backtrace; 
unsigned int __read_mostly hardlockup_panic =

			IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);



/*

 * bitmasks to control what kinds of system info to be printed when

 * hard lockup is detected, it could be task, memory, lock etc.

 * Refer include/linux/sys_info.h for detailed bit definition.

 */

static unsigned long hardlockup_si_mask;



#ifdef CONFIG_SYSFS



static unsigned int hardlockup_count;
@@ -178,11 +186,15 @@ static void watchdog_hardlockup_kick(void) 


void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)

{

	int hardlockup_all_cpu_backtrace;



	if (per_cpu(watchdog_hardlockup_touched, cpu)) {

		per_cpu(watchdog_hardlockup_touched, cpu) = false;

		return;

	}



	hardlockup_all_cpu_backtrace = (hardlockup_si_mask & SYS_INFO_ALL_BT) ?

					1 : sysctl_hardlockup_all_cpu_backtrace;

	/*

	 * Check for a hardlockup by making sure the CPU's timer

	 * interrupt is incrementing. The timer interrupt should have
@@ -205,7 +217,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) 
		 * Prevent multiple hard-lockup reports if one cpu is already

		 * engaged in dumping all cpu back traces.

		 */

		if (sysctl_hardlockup_all_cpu_backtrace) {

		if (hardlockup_all_cpu_backtrace) {

			if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))

				return;

		}
@@ -234,12 +246,13 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) 
			trigger_single_cpu_backtrace(cpu);

		}



		if (sysctl_hardlockup_all_cpu_backtrace) {

		if (hardlockup_all_cpu_backtrace) {

			trigger_allbutcpu_cpu_backtrace(cpu);

			if (!hardlockup_panic)

				clear_bit_unlock(0, &hard_lockup_nmi_warn);

		}



		sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);

		if (hardlockup_panic)

			nmi_panic(regs, "Hard LOCKUP");
@@ -330,6 +343,13 @@ static void lockup_detector_update_enable(void) 
int __read_mostly sysctl_softlockup_all_cpu_backtrace;

#endif



/*

 * bitmasks to control what kinds of system info to be printed when

 * soft lockup is detected, it could be task, memory, lock etc.

 * Refer include/linux/sys_info.h for detailed bit definition.

 */

static unsigned long softlockup_si_mask;



static struct cpumask watchdog_allowed_mask __read_mostly;



/* Global variables, exported for sysctl */
@@ -746,7 +766,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 
	unsigned long touch_ts, period_ts, now;

	struct pt_regs *regs = get_irq_regs();

	int duration;

	int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;

	int softlockup_all_cpu_backtrace;

	unsigned long flags;



	if (!watchdog_enabled)
@@ -758,6 +778,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 
	if (panic_in_progress())

		return HRTIMER_NORESTART;



	softlockup_all_cpu_backtrace = (softlockup_si_mask & SYS_INFO_ALL_BT) ?

					1 : sysctl_softlockup_all_cpu_backtrace;



	watchdog_hardlockup_kick();



	/* kick the softlockup detector */
@@ -846,6 +869,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) 
		}



		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);

		sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT);

		if (softlockup_panic)

			panic("softlockup: hung tasks");

	}
@@ -1197,6 +1221,13 @@ static const struct ctl_table watchdog_sysctls[] = { 
		.extra1		= SYSCTL_ZERO,

		.extra2		= SYSCTL_ONE,

	},

	{

		.procname	= "softlockup_sys_info",

		.data		= &softlockup_si_mask,

		.maxlen         = sizeof(softlockup_si_mask),

		.mode		= 0644,

		.proc_handler	= sysctl_sys_info_handler,

	},

#ifdef CONFIG_SMP

	{

		.procname	= "softlockup_all_cpu_backtrace",
@@ -1219,6 +1250,13 @@ static const struct ctl_table watchdog_sysctls[] = { 
		.extra1		= SYSCTL_ZERO,

		.extra2		= SYSCTL_ONE,

	},

	{

		.procname	= "hardlockup_sys_info",

		.data		= &hardlockup_si_mask,

		.maxlen         = sizeof(hardlockup_si_mask),

		.mode		= 0644,

		.proc_handler	= sysctl_sys_info_handler,

	},

#ifdef CONFIG_SMP

	{

		.procname	= "hardlockup_all_cpu_backtrace",