Commit 12117f33 authored by Steven Rostedt (VMware)'s avatar Steven Rostedt (VMware) Committed by Steven Rostedt (Google)
Browse files

function_graph: Move set_graph_function tests to shadow stack global var 

The use of the task->trace_recursion for the logic used for the
set_graph_function was a bit of an abuse of that variable. Now that there
exists global vars that are per stack for registered graph traces, use that
instead.

Link: https://lore.kernel.org/linux-trace-kernel/171509105520.162236.10339831553995971290.stgit@devnote2
Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.472955399@goodmis.org



Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Florent Revest <revest@chromium.org>
Cc: Martin KaFai Lau <martin.lau@linux.dev>
Cc: bpf <bpf@vger.kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Alan Maguire <alan.maguire@oracle.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Guo Ren <guoren@kernel.org>
Reviewed-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: default avatarSteven Rostedt (VMware) <rostedt@goodmis.org>
Signed-off-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
parent 4497412a

include/linux/trace_recursion.h

+1 −4
Original line number Diff line number Diff line
@@ -44,9 +44,6 @@ enum { 
 */

	TRACE_IRQ_BIT,



	/* Set if the function is in the set_graph_function file */

	TRACE_GRAPH_BIT,



	/*

	 * In the very unlikely case that an interrupt came in

	 * at a start of graph tracing, and we want to trace
@@ -60,7 +57,7 @@ enum { 
	 * that preempted a softirq start of a function that

	 * preempted normal context!!!! Luckily, it can't be

	 * greater than 3, so the next two bits are a mask

	 * of what the depth is when we set TRACE_GRAPH_BIT

	 * of what the depth is when we set TRACE_GRAPH_FL

	 */



	TRACE_GRAPH_DEPTH_START_BIT,

kernel/trace/trace.h

+21 −11
Original line number Diff line number Diff line
@@ -898,11 +898,16 @@ extern void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops 
extern int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops);

extern void free_fgraph_ops(struct trace_array *tr);



enum {

	TRACE_GRAPH_FL		= 1,

};



#ifdef CONFIG_DYNAMIC_FTRACE

extern struct ftrace_hash __rcu *ftrace_graph_hash;

extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash;



static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)

static inline int

ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace)

{

	unsigned long addr = trace->func;

	int ret = 0;
@@ -924,12 +929,11 @@ static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace) 
	}



	if (ftrace_lookup_ip(hash, addr)) {



		/*

		 * This needs to be cleared on the return functions

		 * when the depth is zero.

		 */

		trace_recursion_set(TRACE_GRAPH_BIT);

		*task_var |= TRACE_GRAPH_FL;

		trace_recursion_set_depth(trace->depth);



		/*
@@ -949,11 +953,14 @@ static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace) 
	return ret;

}



static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)

static inline void

ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace)

{

	if (trace_recursion_test(TRACE_GRAPH_BIT) &&

	unsigned long *task_var = fgraph_get_task_var(gops);



	if ((*task_var & TRACE_GRAPH_FL) &&

	    trace->depth == trace_recursion_depth())

		trace_recursion_clear(TRACE_GRAPH_BIT);

		*task_var &= ~TRACE_GRAPH_FL;

}



static inline int ftrace_graph_notrace_addr(unsigned long addr)
@@ -979,7 +986,7 @@ static inline int ftrace_graph_notrace_addr(unsigned long addr) 
	return ret;

}

#else

static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)

static inline int ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace)

{

	return 1;

}
@@ -988,17 +995,20 @@ static inline int ftrace_graph_notrace_addr(unsigned long addr) 
{

	return 0;

}

static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)

static inline void ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace)

{ }

#endif /* CONFIG_DYNAMIC_FTRACE */



extern unsigned int fgraph_max_depth;



static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace)

static inline bool

ftrace_graph_ignore_func(struct fgraph_ops *gops, struct ftrace_graph_ent *trace)

{

	unsigned long *task_var = fgraph_get_task_var(gops);



	/* trace it when it is-nested-in or is a function enabled. */

	return !(trace_recursion_test(TRACE_GRAPH_BIT) ||

		 ftrace_graph_addr(trace)) ||

	return !((*task_var & TRACE_GRAPH_FL) ||

		 ftrace_graph_addr(task_var, trace)) ||

		(trace->depth < 0) ||

		(fgraph_max_depth && trace->depth >= fgraph_max_depth);

}

kernel/trace/trace_functions_graph.c

+3 −3
Original line number Diff line number Diff line
@@ -160,7 +160,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace, 
	if (!ftrace_trace_task(tr))

		return 0;



	if (ftrace_graph_ignore_func(trace))

	if (ftrace_graph_ignore_func(gops, trace))

		return 0;



	if (ftrace_graph_ignore_irqs())
@@ -247,7 +247,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace, 
	long disabled;

	int cpu;



	ftrace_graph_addr_finish(trace);

	ftrace_graph_addr_finish(gops, trace);



	if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) {

		trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT);
@@ -269,7 +269,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace, 
static void trace_graph_thresh_return(struct ftrace_graph_ret *trace,

				      struct fgraph_ops *gops)

{

	ftrace_graph_addr_finish(trace);

	ftrace_graph_addr_finish(gops, trace);



	if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) {

		trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT);

kernel/trace/trace_irqsoff.c

+2 −2
Original line number Diff line number Diff line
@@ -184,7 +184,7 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace, 
	unsigned int trace_ctx;

	int ret;



	if (ftrace_graph_ignore_func(trace))

	if (ftrace_graph_ignore_func(gops, trace))

		return 0;

	/*

	 * Do not trace a function if it's filtered by set_graph_notrace.
@@ -214,7 +214,7 @@ static void irqsoff_graph_return(struct ftrace_graph_ret *trace, 
	unsigned long flags;

	unsigned int trace_ctx;



	ftrace_graph_addr_finish(trace);

	ftrace_graph_addr_finish(gops, trace);



	if (!func_prolog_dec(tr, &data, &flags))

		return;

kernel/trace/trace_sched_wakeup.c

+2 −2
Original line number Diff line number Diff line
@@ -120,7 +120,7 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace, 
	unsigned int trace_ctx;

	int ret = 0;



	if (ftrace_graph_ignore_func(trace))

	if (ftrace_graph_ignore_func(gops, trace))

		return 0;

	/*

	 * Do not trace a function if it's filtered by set_graph_notrace.
@@ -149,7 +149,7 @@ static void wakeup_graph_return(struct ftrace_graph_ret *trace, 
	struct trace_array_cpu *data;

	unsigned int trace_ctx;



	ftrace_graph_addr_finish(trace);

	ftrace_graph_addr_finish(gops, trace);



	if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))

		return;