Commit 73c12f20 authored by Masami Hiramatsu (Google)'s avatar Masami Hiramatsu (Google)
Browse files

kprobes: Use dedicated kthread for kprobe optimizer 

Instead of using generic workqueue, use a dedicated kthread for optimizing
kprobes, because it can wait (sleep) for a long time inside the process
by synchronize_rcu_task(). This means other works can be stopped until it
finishes.

Link: https://lore.kernel.org/all/176970170302.114949.5175231591310436910.stgit@devnote2/



Suggested-by: default avatarSteven Rostedt <rostedt@goodmis.org>
Signed-off-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
parent b8121b9c

kernel/kprobes.c

+86 −20
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ 
#include <linux/debugfs.h>

#include <linux/sysctl.h>

#include <linux/kdebug.h>

#include <linux/kthread.h>

#include <linux/memory.h>

#include <linux/ftrace.h>

#include <linux/cpu.h>
@@ -40,6 +41,7 @@ 
#include <linux/perf_event.h>

#include <linux/execmem.h>

#include <linux/cleanup.h>

#include <linux/wait.h>



#include <asm/sections.h>

#include <asm/cacheflush.h>
@@ -514,9 +516,18 @@ static LIST_HEAD(optimizing_list); 
static LIST_HEAD(unoptimizing_list);

static LIST_HEAD(freeing_list);



static void kprobe_optimizer(struct work_struct *work);

static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);

static void optimize_kprobe(struct kprobe *p);

static struct task_struct *kprobe_optimizer_task;

static wait_queue_head_t kprobe_optimizer_wait;

static atomic_t optimizer_state;

enum {

	OPTIMIZER_ST_IDLE = 0,

	OPTIMIZER_ST_KICKED = 1,

	OPTIMIZER_ST_FLUSHING = 2,

};



static DECLARE_COMPLETION(optimizer_completion);



#define OPTIMIZE_DELAY 5



/*
@@ -609,14 +620,10 @@ static void do_free_cleaned_kprobes(void) 
	}

}



/* Start optimizer after OPTIMIZE_DELAY passed */

static void kick_kprobe_optimizer(void)

{

	schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);

}

static void kick_kprobe_optimizer(void);



/* Kprobe jump optimizer */

static void kprobe_optimizer(struct work_struct *work)

static void kprobe_optimizer(void)

{

	guard(mutex)(&kprobe_mutex);
@@ -647,9 +654,53 @@ static void kprobe_optimizer(struct work_struct *work) 
		do_free_cleaned_kprobes();

	}



	/* Step 5: Kick optimizer again if needed */

	/* Step 5: Kick optimizer again if needed. But if there is a flush requested, */

	if (completion_done(&optimizer_completion))

		complete(&optimizer_completion);



	if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))

		kick_kprobe_optimizer();

		kick_kprobe_optimizer();	/*normal kick*/

}



static int kprobe_optimizer_thread(void *data)

{

	while (!kthread_should_stop()) {

		/* To avoid hung_task, wait in interruptible state. */

		wait_event_interruptible(kprobe_optimizer_wait,

			   atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE ||

			   kthread_should_stop());



		if (kthread_should_stop())

			break;



		/*

		 * If it was a normal kick, wait for OPTIMIZE_DELAY.

		 * This wait can be interrupted by a flush request.

		 */

		if (atomic_read(&optimizer_state) == 1)

			wait_event_interruptible_timeout(

				kprobe_optimizer_wait,

				atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING ||

				kthread_should_stop(),

				OPTIMIZE_DELAY);



		if (kthread_should_stop())

			break;



		atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);



		kprobe_optimizer();

	}

	return 0;

}



/* Start optimizer after OPTIMIZE_DELAY passed */

static void kick_kprobe_optimizer(void)

{

	lockdep_assert_held(&kprobe_mutex);

	if (atomic_cmpxchg(&optimizer_state,

		OPTIMIZER_ST_IDLE, OPTIMIZER_ST_KICKED) == OPTIMIZER_ST_IDLE)

		wake_up(&kprobe_optimizer_wait);

}



static void wait_for_kprobe_optimizer_locked(void)
@@ -657,13 +708,17 @@ static void wait_for_kprobe_optimizer_locked(void) 
	lockdep_assert_held(&kprobe_mutex);



	while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {

		mutex_unlock(&kprobe_mutex);



		/* This will also make 'optimizing_work' execute immmediately */

		flush_delayed_work(&optimizing_work);

		/* 'optimizing_work' might not have been queued yet, relax */

		cpu_relax();

		init_completion(&optimizer_completion);

		/*

		 * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's

		 * idle. If it's already kicked, it will see the state change.

		 */

		if (atomic_xchg_acquire(&optimizer_state,

			OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING)

			wake_up(&kprobe_optimizer_wait);



		mutex_unlock(&kprobe_mutex);

		wait_for_completion(&optimizer_completion);

		mutex_lock(&kprobe_mutex);

	}

}
@@ -1016,8 +1071,21 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt) 
	}

}



static void __init init_optprobe(void)

{

#ifdef __ARCH_WANT_KPROBES_INSN_SLOT

	/* Init 'kprobe_optinsn_slots' for allocation */

	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;

#endif



	init_waitqueue_head(&kprobe_optimizer_wait);

	atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);

	kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL,

					    "kprobe-optimizer");

}

#else /* !CONFIG_OPTPROBES */



#define init_optprobe()				do {} while (0)

#define optimize_kprobe(p)			do {} while (0)

#define unoptimize_kprobe(p, f)			do {} while (0)

#define kill_optimized_kprobe(p)		do {} while (0)
@@ -2700,10 +2768,8 @@ static int __init init_kprobes(void) 
	/* By default, kprobes are armed */

	kprobes_all_disarmed = false;



#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)

	/* Init 'kprobe_optinsn_slots' for allocation */

	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;

#endif

	/* Initialize the optimization infrastructure */

	init_optprobe();



	err = arch_init_kprobes();

	if (!err)