Commit fe6f600c authored by Mateusz Guzik's avatar Mateusz Guzik Committed by Andrew Morton
Browse files

exit: combine work under lock in synchronize_group_exit() and coredump_task_exit() 

This reduces single-threaded overhead as it avoids one lock+irq trip on
exit.

It also improves scalability of spawning and killing threads within one
process (just shy of 5% when doing it on 24 cores on my test jig).

Both routines are moved below kcov and kmsan exit, which should be
harmless.

Link: https://lkml.kernel.org/r/20250319195436.1864415-1-mjguzik@gmail.com


Signed-off-by: default avatarMateusz Guzik <mjguzik@gmail.com>
Reviewed-by: default avatarOleg Nesterov <oleg@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 3eff6a3e

kernel/exit.c

+32 −36
Original line number Diff line number Diff line
@@ -415,25 +415,12 @@ kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent) 
	}

}



static void coredump_task_exit(struct task_struct *tsk)

static void coredump_task_exit(struct task_struct *tsk,

			       struct core_state *core_state)

{

	struct core_state *core_state;



	/*

	 * Serialize with any possible pending coredump.

	 * We must hold siglock around checking core_state

	 * and setting PF_POSTCOREDUMP.  The core-inducing thread

	 * will increment ->nr_threads for each thread in the

	 * group without PF_POSTCOREDUMP set.

	 */

	spin_lock_irq(&tsk->sighand->siglock);

	tsk->flags |= PF_POSTCOREDUMP;

	core_state = tsk->signal->core_state;

	spin_unlock_irq(&tsk->sighand->siglock);

	if (core_state) {

	struct core_thread self;



		self.task = current;

	self.task = tsk;

	if (self.task->flags & PF_SIGNALED)

		self.next = xchg(&core_state->dumper.next, &self);

	else
@@ -453,7 +440,6 @@ static void coredump_task_exit(struct task_struct *tsk) 
	}

	__set_current_state(TASK_RUNNING);

}

}



#ifdef CONFIG_MEMCG

/* drops tasklist_lock if succeeds */
@@ -876,6 +862,7 @@ static void synchronize_group_exit(struct task_struct *tsk, long code) 
{

	struct sighand_struct *sighand = tsk->sighand;

	struct signal_struct *signal = tsk->signal;

	struct core_state *core_state;



	spin_lock_irq(&sighand->siglock);

	signal->quick_threads--;
@@ -885,7 +872,19 @@ static void synchronize_group_exit(struct task_struct *tsk, long code) 
		signal->group_exit_code = code;

		signal->group_stop_count = 0;

	}

	/*

	 * Serialize with any possible pending coredump.

	 * We must hold siglock around checking core_state

	 * and setting PF_POSTCOREDUMP.  The core-inducing thread

	 * will increment ->nr_threads for each thread in the

	 * group without PF_POSTCOREDUMP set.

	 */

	tsk->flags |= PF_POSTCOREDUMP;

	core_state = signal->core_state;

	spin_unlock_irq(&sighand->siglock);



	if (unlikely(core_state))

		coredump_task_exit(tsk, core_state);

}



void __noreturn do_exit(long code)
@@ -894,15 +893,12 @@ void __noreturn do_exit(long code) 
	int group_dead;



	WARN_ON(irqs_disabled());



	synchronize_group_exit(tsk, code);



	WARN_ON(tsk->plug);



	kcov_task_exit(tsk);

	kmsan_task_exit(tsk);



	coredump_task_exit(tsk);

	synchronize_group_exit(tsk, code);

	ptrace_event(PTRACE_EVENT_EXIT, code);

	user_events_exit(tsk);