Commit cf9f0f7c authored by Vincent Donnefort's avatar Vincent Donnefort Committed by Steven Rostedt (Google)
Browse files

tracing: Allow user-space mapping of the ring-buffer 

Currently, user-space extracts data from the ring-buffer via splice,
which is handy for storage or network sharing. However, due to splice
limitations, it is imposible to do real-time analysis without a copy.

A solution for that problem is to let the user-space map the ring-buffer
directly.

The mapping is exposed via the per-CPU file trace_pipe_raw. The first
element of the mapping is the meta-page. It is followed by each
subbuffer constituting the ring-buffer, ordered by their unique page ID:

  * Meta-page -- include/uapi/linux/trace_mmap.h for a description
  * Subbuf ID 0
  * Subbuf ID 1
     ...

It is therefore easy to translate a subbuf ID into an offset in the
mapping:

  reader_id = meta->reader->id;
  reader_offset = meta->meta_page_size + reader_id * meta->subbuf_size;

When new data is available, the mapper must call a newly introduced ioctl:
TRACE_MMAP_IOCTL_GET_READER. This will update the Meta-page reader ID to
point to the next reader containing unread data.

Mapping will prevent snapshot and buffer size modifications.

Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-4-vdonnefort@google.com



CC: <linux-mm@kvack.org>
Signed-off-by: default avatarVincent Donnefort <vdonnefort@google.com>
Signed-off-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
parent 117c3920

include/uapi/linux/trace_mmap.h

+2 −0
Original line number Diff line number Diff line
@@ -43,4 +43,6 @@ struct trace_buffer_meta { 
	__u64	Reserved2;

};



#define TRACE_MMAP_IOCTL_GET_READER		_IO('T', 0x1)



#endif /* _TRACE_MMAP_H_ */

kernel/trace/trace.c

+99 −5
Original line number Diff line number Diff line
@@ -1191,6 +1191,12 @@ static void tracing_snapshot_instance_cond(struct trace_array *tr, 
		return;

	}



	if (tr->mapped) {

		trace_array_puts(tr, "*** BUFFER MEMORY MAPPED ***\n");

		trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n");

		return;

	}



	local_irq_save(flags);

	update_max_tr(tr, current, smp_processor_id(), cond_data);

	local_irq_restore(flags);
@@ -1323,7 +1329,7 @@ static int tracing_arm_snapshot_locked(struct trace_array *tr) 
	lockdep_assert_held(&trace_types_lock);



	spin_lock(&tr->snapshot_trigger_lock);

	if (tr->snapshot == UINT_MAX) {

	if (tr->snapshot == UINT_MAX || tr->mapped) {

		spin_unlock(&tr->snapshot_trigger_lock);

		return -EBUSY;

	}
@@ -8194,15 +8200,32 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, 
	return ret;

}



/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */

static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)

{

	struct ftrace_buffer_info *info = file->private_data;

	struct trace_iterator *iter = &info->iter;

	int err;



	if (cmd)

		return -ENOIOCTLCMD;

	if (cmd == TRACE_MMAP_IOCTL_GET_READER) {

		if (!(file->f_flags & O_NONBLOCK)) {

			err = ring_buffer_wait(iter->array_buffer->buffer,

					       iter->cpu_file,

					       iter->tr->buffer_percent,

					       NULL, NULL);

			if (err)

				return err;

		}



		return ring_buffer_map_get_reader(iter->array_buffer->buffer,

						  iter->cpu_file);

	} else if (cmd) {

		return -ENOTTY;

	}



	/*

	 * An ioctl call with cmd 0 to the ring buffer file will wake up all

	 * waiters

	 */

	mutex_lock(&trace_types_lock);



	/* Make sure the waiters see the new wait_index */
@@ -8214,6 +8237,76 @@ static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned 
	return 0;

}



#ifdef CONFIG_TRACER_MAX_TRACE

static int get_snapshot_map(struct trace_array *tr)

{

	int err = 0;



	/*

	 * Called with mmap_lock held. lockdep would be unhappy if we would now

	 * take trace_types_lock. Instead use the specific

	 * snapshot_trigger_lock.

	 */

	spin_lock(&tr->snapshot_trigger_lock);



	if (tr->snapshot || tr->mapped == UINT_MAX)

		err = -EBUSY;

	else

		tr->mapped++;



	spin_unlock(&tr->snapshot_trigger_lock);



	/* Wait for update_max_tr() to observe iter->tr->mapped */

	if (tr->mapped == 1)

		synchronize_rcu();



	return err;



}

static void put_snapshot_map(struct trace_array *tr)

{

	spin_lock(&tr->snapshot_trigger_lock);

	if (!WARN_ON(!tr->mapped))

		tr->mapped--;

	spin_unlock(&tr->snapshot_trigger_lock);

}

#else

static inline int get_snapshot_map(struct trace_array *tr) { return 0; }

static inline void put_snapshot_map(struct trace_array *tr) { }

#endif



static void tracing_buffers_mmap_close(struct vm_area_struct *vma)

{

	struct ftrace_buffer_info *info = vma->vm_file->private_data;

	struct trace_iterator *iter = &info->iter;



	WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file));

	put_snapshot_map(iter->tr);

}



static const struct vm_operations_struct tracing_buffers_vmops = {

	.close		= tracing_buffers_mmap_close,

};



static int tracing_buffers_mmap(struct file *filp, struct vm_area_struct *vma)

{

	struct ftrace_buffer_info *info = filp->private_data;

	struct trace_iterator *iter = &info->iter;

	int ret = 0;



	ret = get_snapshot_map(iter->tr);

	if (ret)

		return ret;



	ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma);

	if (ret)

		put_snapshot_map(iter->tr);



	vma->vm_ops = &tracing_buffers_vmops;



	return ret;

}



static const struct file_operations tracing_buffers_fops = {

	.open		= tracing_buffers_open,

	.read		= tracing_buffers_read,
@@ -8223,6 +8316,7 @@ static const struct file_operations tracing_buffers_fops = { 
	.splice_read	= tracing_buffers_splice_read,

	.unlocked_ioctl = tracing_buffers_ioctl,

	.llseek		= no_llseek,

	.mmap		= tracing_buffers_mmap,

};



static ssize_t

kernel/trace/trace.h

+1 −0
Original line number Diff line number Diff line
@@ -336,6 +336,7 @@ struct trace_array { 
	bool			allocated_snapshot;

	spinlock_t		snapshot_trigger_lock;

	unsigned int		snapshot;

	unsigned int		mapped;

	unsigned long		max_latency;

#ifdef CONFIG_FSNOTIFY

	struct dentry		*d_max_latency;