Commit 3eaea21b authored by Andrii Nakryiko's avatar Andrii Nakryiko Committed by Masami Hiramatsu (Google)
Browse files

uprobes: encapsulate preparation of uprobe args buffer 

Move the logic of fetching temporary per-CPU uprobe buffer and storing
uprobes args into it to a new helper function. Store data size as part
of this buffer, simplifying interfaces a bit, as now we only pass single
uprobe_cpu_buffer reference around, instead of pointer + dsize.

This logic was duplicated across uprobe_dispatcher and uretprobe_dispatcher,
and now will be centralized. All this is also in preparation to make
this uprobe_cpu_buffer handling logic optional in the next patch.

Link: https://lore.kernel.org/all/20240318181728.2795838-2-andrii@kernel.org/


[Masami: update for v6.9-rc3 kernel]

Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Reviewed-by: default avatarJiri Olsa <jolsa@kernel.org>
Acked-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
parent e67572cd

kernel/trace/trace_uprobe.c

+41 −37
Original line number Diff line number Diff line
@@ -854,6 +854,7 @@ static const struct file_operations uprobe_profile_ops = { 
struct uprobe_cpu_buffer {

	struct mutex mutex;

	void *buf;

	int dsize;

};

static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;

static int uprobe_buffer_refcnt;
@@ -943,9 +944,26 @@ static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb) 
	mutex_unlock(&ucb->mutex);

}



static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu,

						       struct pt_regs *regs)

{

	struct uprobe_cpu_buffer *ucb;

	int dsize, esize;



	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));

	dsize = __get_data_size(&tu->tp, regs, NULL);



	ucb = uprobe_buffer_get();

	ucb->dsize = tu->tp.size + dsize;



	store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);



	return ucb;

}



static void __uprobe_trace_func(struct trace_uprobe *tu,

				unsigned long func, struct pt_regs *regs,

				struct uprobe_cpu_buffer *ucb, int dsize,

				struct uprobe_cpu_buffer *ucb,

				struct trace_event_file *trace_file)

{

	struct uprobe_trace_entry_head *entry;
@@ -956,14 +974,14 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, 


	WARN_ON(call != trace_file->event_call);



	if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))

	if (WARN_ON_ONCE(ucb->dsize > PAGE_SIZE))

		return;



	if (trace_trigger_soft_disabled(trace_file))

		return;



	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));

	size = esize + tu->tp.size + dsize;

	size = esize + ucb->dsize;

	entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);

	if (!entry)

		return;
@@ -977,14 +995,14 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, 
		data = DATAOF_TRACE_ENTRY(entry, false);

	}



	memcpy(data, ucb->buf, tu->tp.size + dsize);

	memcpy(data, ucb->buf, ucb->dsize);



	trace_event_buffer_commit(&fbuffer);

}



/* uprobe handler */

static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,

			     struct uprobe_cpu_buffer *ucb, int dsize)

			     struct uprobe_cpu_buffer *ucb)

{

	struct event_file_link *link;
@@ -993,7 +1011,7 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, 


	rcu_read_lock();

	trace_probe_for_each_link_rcu(link, &tu->tp)

		__uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);

		__uprobe_trace_func(tu, 0, regs, ucb, link->file);

	rcu_read_unlock();



	return 0;
@@ -1001,13 +1019,13 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, 


static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,

				 struct pt_regs *regs,

				 struct uprobe_cpu_buffer *ucb, int dsize)

				 struct uprobe_cpu_buffer *ucb)

{

	struct event_file_link *link;



	rcu_read_lock();

	trace_probe_for_each_link_rcu(link, &tu->tp)

		__uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);

		__uprobe_trace_func(tu, func, regs, ucb, link->file);

	rcu_read_unlock();

}
@@ -1335,7 +1353,7 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc, 


static void __uprobe_perf_func(struct trace_uprobe *tu,

			       unsigned long func, struct pt_regs *regs,

			       struct uprobe_cpu_buffer *ucb, int dsize)

			       struct uprobe_cpu_buffer *ucb)

{

	struct trace_event_call *call = trace_probe_event_call(&tu->tp);

	struct uprobe_trace_entry_head *entry;
@@ -1356,7 +1374,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, 


	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));



	size = esize + tu->tp.size + dsize;

	size = esize + ucb->dsize;

	size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);

	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))

		return;
@@ -1379,13 +1397,10 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, 
		data = DATAOF_TRACE_ENTRY(entry, false);

	}



	memcpy(data, ucb->buf, tu->tp.size + dsize);

	memcpy(data, ucb->buf, ucb->dsize);



	if (size - esize > tu->tp.size + dsize) {

		int len = tu->tp.size + dsize;



		memset(data + len, 0, size - esize - len);

	}

	if (size - esize > ucb->dsize)

		memset(data + ucb->dsize, 0, size - esize - ucb->dsize);



	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,

			      head, NULL);
@@ -1395,21 +1410,21 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, 


/* uprobe profile handler */

static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,

			    struct uprobe_cpu_buffer *ucb, int dsize)

			    struct uprobe_cpu_buffer *ucb)

{

	if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))

		return UPROBE_HANDLER_REMOVE;



	if (!is_ret_probe(tu))

		__uprobe_perf_func(tu, 0, regs, ucb, dsize);

		__uprobe_perf_func(tu, 0, regs, ucb);

	return 0;

}



static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,

				struct pt_regs *regs,

				struct uprobe_cpu_buffer *ucb, int dsize)

				struct uprobe_cpu_buffer *ucb)

{

	__uprobe_perf_func(tu, func, regs, ucb, dsize);

	__uprobe_perf_func(tu, func, regs, ucb);

}



int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
@@ -1475,10 +1490,8 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) 
	struct trace_uprobe *tu;

	struct uprobe_dispatch_data udd;

	struct uprobe_cpu_buffer *ucb;

	int dsize, esize;

	int ret = 0;





	tu = container_of(con, struct trace_uprobe, consumer);

	tu->nhit++;
@@ -1490,18 +1503,14 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) 
	if (WARN_ON_ONCE(!uprobe_cpu_buffer))

		return 0;



	dsize = __get_data_size(&tu->tp, regs, NULL);

	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));



	ucb = uprobe_buffer_get();

	store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);

	ucb = prepare_uprobe_buffer(tu, regs);



	if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))

		ret |= uprobe_trace_func(tu, regs, ucb, dsize);

		ret |= uprobe_trace_func(tu, regs, ucb);



#ifdef CONFIG_PERF_EVENTS

	if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))

		ret |= uprobe_perf_func(tu, regs, ucb, dsize);

		ret |= uprobe_perf_func(tu, regs, ucb);

#endif

	uprobe_buffer_put(ucb);

	return ret;
@@ -1513,7 +1522,6 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, 
	struct trace_uprobe *tu;

	struct uprobe_dispatch_data udd;

	struct uprobe_cpu_buffer *ucb;

	int dsize, esize;



	tu = container_of(con, struct trace_uprobe, consumer);
@@ -1525,18 +1533,14 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, 
	if (WARN_ON_ONCE(!uprobe_cpu_buffer))

		return 0;



	dsize = __get_data_size(&tu->tp, regs, NULL);

	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));



	ucb = uprobe_buffer_get();

	store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);

	ucb = prepare_uprobe_buffer(tu, regs);



	if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))

		uretprobe_trace_func(tu, func, regs, ucb, dsize);

		uretprobe_trace_func(tu, func, regs, ucb);



#ifdef CONFIG_PERF_EVENTS

	if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))

		uretprobe_perf_func(tu, func, regs, ucb, dsize);

		uretprobe_perf_func(tu, func, regs, ucb);

#endif

	uprobe_buffer_put(ucb);

	return 0;