Commit 5159938e authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'probes-v6.12' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace 

Pull probes updates from Masami Hiramatsu:

 - uprobes: make trace_uprobe->nhit counter a per-CPU one

   This makes uprobe event's hit counter per-CPU for improving
   scalability on multi-core environment

 - kprobes: Remove obsoleted declaration for init_test_probes

   Remove unused init_test_probes() from header

 - Raw tracepoint probe supports raw tracepoint events on modules:
     - add a function for iterating over all tracepoints in all modules
     - add a function for iterating over tracepoints in a module
     - support raw tracepoint events on modules
     - support raw tracepoints on future loaded modules
     - add a test for tracepoint events on modules"

* tag 'probes-v6.12' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace:
  sefltests/tracing: Add a test for tracepoint events on modules
  tracing/fprobe: Support raw tracepoints on future loaded modules
  tracing/fprobe: Support raw tracepoint events on modules
  tracepoint: Support iterating tracepoints in a loading module
  tracepoint: Support iterating over tracepoints on modules
  kprobes: Remove obsoleted declaration for init_test_probes
  uprobes: turn trace_uprobe's nhit counter to be per-CPU one
parents 0181f8c8 4e78dd6b

include/linux/kprobes.h

+0 −9
Original line number Diff line number Diff line
@@ -269,15 +269,6 @@ extern unsigned long __stop_kprobe_blacklist[]; 


extern struct kretprobe_blackpoint kretprobe_blacklist[];



#ifdef CONFIG_KPROBES_SANITY_TEST

extern int init_test_probes(void);

#else /* !CONFIG_KPROBES_SANITY_TEST */

static inline int init_test_probes(void)

{

	return 0;

}

#endif /* CONFIG_KPROBES_SANITY_TEST */



extern int arch_prepare_kprobe(struct kprobe *p);

extern void arch_arm_kprobe(struct kprobe *p);

extern void arch_disarm_kprobe(struct kprobe *p);

include/linux/tracepoint.h

+20 −0
Original line number Diff line number Diff line
@@ -64,6 +64,13 @@ struct tp_module { 
bool trace_module_has_bad_taint(struct module *mod);

extern int register_tracepoint_module_notifier(struct notifier_block *nb);

extern int unregister_tracepoint_module_notifier(struct notifier_block *nb);

void for_each_module_tracepoint(void (*fct)(struct tracepoint *,

					struct module *, void *),

				void *priv);

void for_each_tracepoint_in_module(struct module *,

				   void (*fct)(struct tracepoint *,

					struct module *, void *),

				   void *priv);

#else

static inline bool trace_module_has_bad_taint(struct module *mod)

{
@@ -79,6 +86,19 @@ int unregister_tracepoint_module_notifier(struct notifier_block *nb) 
{

	return 0;

}

static inline

void for_each_module_tracepoint(void (*fct)(struct tracepoint *,

					struct module *, void *),

				void *priv)

{

}

static inline

void for_each_tracepoint_in_module(struct module *mod,

				   void (*fct)(struct tracepoint *,

					struct module *, void *),

				   void *priv)

{

}

#endif /* CONFIG_MODULES */



/*

kernel/trace/trace_fprobe.c

+130 −49
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#define FPROBE_EVENT_SYSTEM "fprobes"

#define TRACEPOINT_EVENT_SYSTEM "tracepoints"

#define RETHOOK_MAXACTIVE_MAX 4096

#define TRACEPOINT_STUB ERR_PTR(-ENOENT)



static int trace_fprobe_create(const char *raw_command);

static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
@@ -385,6 +386,7 @@ static struct trace_fprobe *alloc_trace_fprobe(const char *group, 
					       const char *event,

					       const char *symbol,

					       struct tracepoint *tpoint,

					       struct module *mod,

					       int maxactive,

					       int nargs, bool is_return)

{
@@ -405,6 +407,7 @@ static struct trace_fprobe *alloc_trace_fprobe(const char *group, 
		tf->fp.entry_handler = fentry_dispatcher;



	tf->tpoint = tpoint;

	tf->mod = mod;

	tf->fp.nr_maxactive = maxactive;



	ret = trace_probe_init(&tf->tp, event, group, false, nargs);
@@ -672,6 +675,24 @@ static int unregister_fprobe_event(struct trace_fprobe *tf) 
	return trace_probe_unregister_event_call(&tf->tp);

}



static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)

{

	struct tracepoint *tpoint = tf->tpoint;

	unsigned long ip = (unsigned long)tpoint->probestub;

	int ret;



	/*

	 * Here, we do 2 steps to enable fprobe on a tracepoint.

	 * At first, put __probestub_##TP function on the tracepoint

	 * and put a fprobe on the stub function.

	 */

	ret = tracepoint_probe_register_prio_may_exist(tpoint,

				tpoint->probestub, NULL, 0);

	if (ret < 0)

		return ret;

	return register_fprobe_ips(&tf->fp, &ip, 1);

}



/* Internal register function - just handle fprobe and flags */

static int __register_trace_fprobe(struct trace_fprobe *tf)

{
@@ -698,18 +719,12 @@ static int __register_trace_fprobe(struct trace_fprobe *tf) 
		tf->fp.flags |= FPROBE_FL_DISABLED;



	if (trace_fprobe_is_tracepoint(tf)) {

		struct tracepoint *tpoint = tf->tpoint;

		unsigned long ip = (unsigned long)tpoint->probestub;

		/*

		 * Here, we do 2 steps to enable fprobe on a tracepoint.

		 * At first, put __probestub_##TP function on the tracepoint

		 * and put a fprobe on the stub function.

		 */

		ret = tracepoint_probe_register_prio_may_exist(tpoint,

					tpoint->probestub, NULL, 0);

		if (ret < 0)

			return ret;

		return register_fprobe_ips(&tf->fp, &ip, 1);



		/* This tracepoint is not loaded yet */

		if (tf->tpoint == TRACEPOINT_STUB)

			return 0;



		return __regsiter_tracepoint_fprobe(tf);

	}



	/* TODO: handle filter, nofilter or symbol list */
@@ -862,20 +877,106 @@ static int register_trace_fprobe(struct trace_fprobe *tf) 
	return ret;

}



struct __find_tracepoint_cb_data {

	const char *tp_name;

	struct tracepoint *tpoint;

	struct module *mod;

};



static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv)

{

	struct __find_tracepoint_cb_data *data = priv;



	if (!data->tpoint && !strcmp(data->tp_name, tp->name)) {

		data->tpoint = tp;

		if (!data->mod) {

			data->mod = mod;

			if (!try_module_get(data->mod)) {

				data->tpoint = NULL;

				data->mod = NULL;

			}

		}

	}

}



static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)

{

	struct __find_tracepoint_cb_data *data = priv;



	if (!data->tpoint && !strcmp(data->tp_name, tp->name))

		data->tpoint = tp;

}



/*

 * Find a tracepoint from kernel and module. If the tracepoint is in a module,

 * this increments the module refcount to prevent unloading until the

 * trace_fprobe is registered to the list. After registering the trace_fprobe

 * on the trace_fprobe list, the module refcount is decremented because

 * tracepoint_probe_module_cb will handle it.

 */

static struct tracepoint *find_tracepoint(const char *tp_name,

					  struct module **tp_mod)

{

	struct __find_tracepoint_cb_data data = {

		.tp_name = tp_name,

		.mod = NULL,

	};



	for_each_kernel_tracepoint(__find_tracepoint_cb, &data);



	if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) {

		for_each_module_tracepoint(__find_tracepoint_module_cb, &data);

		*tp_mod = data.mod;

	}



	return data.tpoint;

}



#ifdef CONFIG_MODULES

static void reenable_trace_fprobe(struct trace_fprobe *tf)

{

	struct trace_probe *tp = &tf->tp;



	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {

		__enable_trace_fprobe(tf);

	}

}



static struct tracepoint *find_tracepoint_in_module(struct module *mod,

						    const char *tp_name)

{

	struct __find_tracepoint_cb_data data = {

		.tp_name = tp_name,

		.mod = mod,

	};



	for_each_tracepoint_in_module(mod, __find_tracepoint_module_cb, &data);

	return data.tpoint;

}



static int __tracepoint_probe_module_cb(struct notifier_block *self,

					unsigned long val, void *data)

{

	struct tp_module *tp_mod = data;

	struct tracepoint *tpoint;

	struct trace_fprobe *tf;

	struct dyn_event *pos;



	if (val != MODULE_STATE_GOING)

	if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)

		return NOTIFY_DONE;



	mutex_lock(&event_mutex);

	for_each_trace_fprobe(tf, pos) {

		if (tp_mod->mod == tf->mod) {

		if (val == MODULE_STATE_COMING && tf->tpoint == TRACEPOINT_STUB) {

			tpoint = find_tracepoint_in_module(tp_mod->mod, tf->symbol);

			if (tpoint) {

				tf->tpoint = tpoint;

				tf->mod = tp_mod->mod;

				if (!WARN_ON_ONCE(__regsiter_tracepoint_fprobe(tf)) &&

				    trace_probe_is_enabled(&tf->tp))

					reenable_trace_fprobe(tf);

			}

		} else if (val == MODULE_STATE_GOING && tp_mod->mod == tf->mod) {

			tracepoint_probe_unregister(tf->tpoint,

					tf->tpoint->probestub, NULL);

			tf->tpoint = NULL;
@@ -892,30 +993,6 @@ static struct notifier_block tracepoint_module_nb = { 
};

#endif /* CONFIG_MODULES */



struct __find_tracepoint_cb_data {

	const char *tp_name;

	struct tracepoint *tpoint;

};



static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)

{

	struct __find_tracepoint_cb_data *data = priv;



	if (!data->tpoint && !strcmp(data->tp_name, tp->name))

		data->tpoint = tp;

}



static struct tracepoint *find_tracepoint(const char *tp_name)

{

	struct __find_tracepoint_cb_data data = {

		.tp_name = tp_name,

	};



	for_each_kernel_tracepoint(__find_tracepoint_cb, &data);



	return data.tpoint;

}



static int parse_symbol_and_return(int argc, const char *argv[],

				   char **symbol, bool *is_return,

				   bool is_tracepoint)
@@ -996,6 +1073,7 @@ static int __trace_fprobe_create(int argc, const char *argv[]) 
	char abuf[MAX_BTF_ARGS_LEN];

	char *dbuf = NULL;

	bool is_tracepoint = false;

	struct module *tp_mod = NULL;

	struct tracepoint *tpoint = NULL;

	struct traceprobe_parse_context ctx = {

		.flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE,
@@ -1080,15 +1158,20 @@ static int __trace_fprobe_create(int argc, const char *argv[]) 


	if (is_tracepoint) {

		ctx.flags |= TPARG_FL_TPOINT;

		tpoint = find_tracepoint(symbol);

		if (!tpoint) {

		tpoint = find_tracepoint(symbol, &tp_mod);

		if (tpoint) {

			ctx.funcname = kallsyms_lookup(

				(unsigned long)tpoint->probestub,

				NULL, NULL, NULL, sbuf);

		} else if (IS_ENABLED(CONFIG_MODULES)) {

				/* This *may* be loaded afterwards */

				tpoint = TRACEPOINT_STUB;

				ctx.funcname = symbol;

		} else {

			trace_probe_log_set_index(1);

			trace_probe_log_err(0, NO_TRACEPOINT);

			goto parse_error;

		}

		ctx.funcname = kallsyms_lookup(

				(unsigned long)tpoint->probestub,

				NULL, NULL, NULL, sbuf);

	} else

		ctx.funcname = symbol;
@@ -1110,8 +1193,8 @@ static int __trace_fprobe_create(int argc, const char *argv[]) 
		goto out;



	/* setup a probe */

	tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive,

				argc, is_return);

	tf = alloc_trace_fprobe(group, event, symbol, tpoint, tp_mod,

				maxactive, argc, is_return);

	if (IS_ERR(tf)) {

		ret = PTR_ERR(tf);

		/* This must return -ENOMEM, else there is a bug */
@@ -1119,10 +1202,6 @@ static int __trace_fprobe_create(int argc, const char *argv[]) 
		goto out;	/* We know tf is not allocated */

	}



	if (is_tracepoint)

		tf->mod = __module_text_address(

				(unsigned long)tf->tpoint->probestub);



	/* parse arguments */

	for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {

		trace_probe_log_set_index(i + 2);
@@ -1155,6 +1234,8 @@ static int __trace_fprobe_create(int argc, const char *argv[]) 
	}



out:

	if (tp_mod)

		module_put(tp_mod);

	traceprobe_finish_parse(&ctx);

	trace_probe_log_clear();

	kfree(new_argv);

kernel/trace/trace_uprobe.c

+21 −3
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include <linux/string.h>

#include <linux/rculist.h>

#include <linux/filter.h>

#include <linux/percpu.h>



#include "trace_dynevent.h"

#include "trace_probe.h"
@@ -62,7 +63,7 @@ struct trace_uprobe { 
	struct uprobe			*uprobe;

	unsigned long			offset;

	unsigned long			ref_ctr_offset;

	unsigned long			nhit;

	unsigned long __percpu		*nhits;

	struct trace_probe		tp;

};
@@ -337,6 +338,12 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) 
	if (!tu)

		return ERR_PTR(-ENOMEM);



	tu->nhits = alloc_percpu(unsigned long);

	if (!tu->nhits) {

		ret = -ENOMEM;

		goto error;

	}



	ret = trace_probe_init(&tu->tp, event, group, true, nargs);

	if (ret < 0)

		goto error;
@@ -349,6 +356,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) 
	return tu;



error:

	free_percpu(tu->nhits);

	kfree(tu);



	return ERR_PTR(ret);
@@ -362,6 +370,7 @@ static void free_trace_uprobe(struct trace_uprobe *tu) 
	path_put(&tu->path);

	trace_probe_cleanup(&tu->tp);

	kfree(tu->filename);

	free_percpu(tu->nhits);

	kfree(tu);

}
@@ -815,13 +824,21 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) 
{

	struct dyn_event *ev = v;

	struct trace_uprobe *tu;

	unsigned long nhits;

	int cpu;



	if (!is_trace_uprobe(ev))

		return 0;



	tu = to_trace_uprobe(ev);



	nhits = 0;

	for_each_possible_cpu(cpu) {

		nhits += per_cpu(*tu->nhits, cpu);

	}



	seq_printf(m, "  %s %-44s %15lu\n", tu->filename,

			trace_probe_name(&tu->tp), tu->nhit);

		   trace_probe_name(&tu->tp), nhits);

	return 0;

}
@@ -1508,7 +1525,8 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) 
	int ret = 0;



	tu = container_of(con, struct trace_uprobe, consumer);

	tu->nhit++;



	this_cpu_inc(*tu->nhits);



	udd.tu = tu;

	udd.bp_addr = instruction_pointer(regs);

kernel/tracepoint.c

+42 −0
Original line number Diff line number Diff line
@@ -735,6 +735,48 @@ static __init int init_tracepoints(void) 
	return ret;

}

__initcall(init_tracepoints);



/**

 * for_each_tracepoint_in_module - iteration on all tracepoints in a module

 * @mod: module

 * @fct: callback

 * @priv: private data

 */

void for_each_tracepoint_in_module(struct module *mod,

				   void (*fct)(struct tracepoint *tp,

				    struct module *mod, void *priv),

				   void *priv)

{

	tracepoint_ptr_t *begin, *end, *iter;



	lockdep_assert_held(&tracepoint_module_list_mutex);



	if (!mod)

		return;



	begin = mod->tracepoints_ptrs;

	end = mod->tracepoints_ptrs + mod->num_tracepoints;



	for (iter = begin; iter < end; iter++)

		fct(tracepoint_ptr_deref(iter), mod, priv);

}



/**

 * for_each_module_tracepoint - iteration on all tracepoints in all modules

 * @fct: callback

 * @priv: private data

 */

void for_each_module_tracepoint(void (*fct)(struct tracepoint *tp,

				 struct module *mod, void *priv),

				void *priv)

{

	struct tp_module *tp_mod;



	mutex_lock(&tracepoint_module_list_mutex);

	list_for_each_entry(tp_mod, &tracepoint_module_list, list)

		for_each_tracepoint_in_module(tp_mod->mod, fct, priv);

	mutex_unlock(&tracepoint_module_list_mutex);

}

#endif /* CONFIG_MODULES */



/**