Commit 5172a777 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'trace-tools-v6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace 

Pull tracing tools updates from Steven Rostedt:

 - Introduce enum timerlat_tracing_mode

   Now that BPF based sampling has been added to timerlat, add an enum
   to represent which mode timerlat is running in

 - Add action on timelat threshold feature

   A new option, --on-threshold, is added, taking an argument that
   further specifies the action. Actions added in this patch are:

     - trace[,file=<filename>]: Saves tracefs buffer, optionally taking a
             filename
     - signal,num=<sig>,pid=<pid>: Sends signal to process. "parent" might
             be specified instead of number to send signal to parent process
     - shell,command=<command>: Execute shell command

 - Allow resuming tracing in timerlat bpf

   rtla-timerlat BPF program uses a global variable stored in a .bss
   section to store whether tracing has been stopped. Map it to allow it
   to resume tracing after it has been stopped

 - Add continue action to timerlat

   Introduce option to resume tracing after a latency threshold
   overflow. The option is implemented as an action named "continue"

 - Add action on end feature to timerlat

   Implement actions on end next to actions on threshold. A new option,
   --on-end is added, parallel to --on-threshold. Instead of being
   executed whenever a latency threshold is reached, it is executed at
   the end of the measurement

 - Have rtla tests check output with grep

   Add argument to the check command in the test suite that takes a
   regular expression that the output of rtla command is checked
   against. This allows testing for specific information in rtla output
   in addition to checking the return value

 - Add tests for timerlat actions

 - Update the documentation for the new features

* tag 'trace-tools-v6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace:
  rtla/tests: Test timerlat -P option using actions
  rtla/tests: Add grep checks for base test cases
  Documentation/rtla: Add actions feature
  rtla/tests: Limit duration to maximum of 10s
  rtla/tests: Add tests for actions
  rtla/tests: Check rtla output with grep
  rtla/timerlat: Add action on end feature
  rtla/timerlat: Add continue action
  rtla/timerlat_bpf: Allow resuming tracing
  rtla/timerlat: Add action on threshold feature
  rtla/timerlat: Introduce enum timerlat_tracing_mode
parents c6439bfa a80db1f8

Documentation/tools/rtla/common_timerlat_options.rst

+64 −0
Original line number Diff line number Diff line
@@ -55,3 +55,67 @@ 
        Set timerlat to run without workload, waiting for the user to dispatch a per-cpu

        task that waits for a new period on the tracing/osnoise/per_cpu/cpu$ID/timerlat_fd.

        See linux/tools/rtla/sample/timerlat_load.py for an example of user-load code.



**--on-threshold** *action*



        Defines an action to be executed when tracing is stopped on a latency threshold

        specified by **-i/--irq** or **-T/--thread**.



        Multiple --on-threshold actions may be specified, and they will be executed in

        the order they are provided. If any action fails, subsequent actions in the list

        will not be executed.



        Supported actions are:



        - *trace[,file=<filename>]*



          Saves trace output, optionally taking a filename. Alternative to -t/--trace.

          Note that nlike -t/--trace, specifying this multiple times will result in

          the trace being saved multiple times.



        - *signal,num=<sig>,pid=<pid>*



          Sends signal to process. "parent" might be specified in place of pid to target

          the parent process of rtla.



        - *shell,command=<command>*



          Execute shell command.



        - *continue*



          Continue tracing after actions are executed instead of stopping.



        Example:



        $ rtla timerlat -T 20 --on-threshold trace

        --on-threshold shell,command="grep ipi_send timerlat_trace.txt"

        --on-threshold signal,num=2,pid=parent



        This will save a trace with the default filename "timerlat_trace.txt", print its

        lines that contain the text "ipi_send" on standard output, and send signal 2

        (SIGINT) to the parent process.



        Performance Considerations:



        For time-sensitive actions, it is recommended to run **rtla timerlat** with BPF

        support and RT priority. Note that due to implementational limitations, actions

        might be delayed up to one second after tracing is stopped if BPF mode is not

        available or disabled.



**--on-end** *action*



        Defines an action to be executed at the end of **rtla timerlat** tracing.



        Multiple --on-end actions can be specified, and they will be executed in the order

        they are provided. If any action fails, subsequent actions in the list will not be

        executed.



        See the documentation for **--on-threshold** for the list of supported actions, with

        the exception that *continue* has no effect.



        Example:



        $ rtla timerlat -d 5s --on-end trace



        This runs rtla timerlat with default options and save trace output at the end.

tools/tracing/rtla/src/Build

+1 −0
Original line number Diff line number Diff line 
rtla-y += trace.o

rtla-y += utils.o

rtla-y += actions.o

rtla-y += osnoise.o

rtla-y += osnoise_top.o

rtla-y += osnoise_hist.o

tools/tracing/rtla/src/actions.c

0 → 100644
+260 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

#include <stdlib.h>

#include <string.h>

#include <signal.h>

#include <unistd.h>



#include "actions.h"

#include "trace.h"

#include "utils.h"



/*

 * actions_init - initialize struct actions

 */

void

actions_init(struct actions *self)

{

	self->size = action_default_size;

	self->list = calloc(self->size, sizeof(struct action));

	self->len = 0;

	self->continue_flag = false;



	memset(&self->present, 0, sizeof(self->present));



	/* This has to be set by the user */

	self->trace_output_inst = NULL;

}



/*

 * actions_destroy - destroy struct actions

 */

void

actions_destroy(struct actions *self)

{

	/* Free any action-specific data */

	for (struct action *action = self->list; action < self->list + self->len; action++) {

		if (action->type == ACTION_SHELL)

			free(action->command);

		if (action->type == ACTION_TRACE_OUTPUT)

			free(action->trace_output);

	}



	/* Free action list */

	free(self->list);

}



/*

 * actions_new - Get pointer to new action

 */

static struct action *

actions_new(struct actions *self)

{

	if (self->size >= self->len) {

		self->size *= 2;

		self->list = realloc(self->list, self->size * sizeof(struct action));

	}



	return &self->list[self->len++];

}



/*

 * actions_add_trace_output - add an action to output trace

 */

int

actions_add_trace_output(struct actions *self, const char *trace_output)

{

	struct action *action = actions_new(self);



	self->present[ACTION_TRACE_OUTPUT] = true;

	action->type = ACTION_TRACE_OUTPUT;

	action->trace_output = calloc(strlen(trace_output) + 1, sizeof(char));

	if (!action->trace_output)

		return -1;

	strcpy(action->trace_output, trace_output);



	return 0;

}



/*

 * actions_add_trace_output - add an action to send signal to a process

 */

int

actions_add_signal(struct actions *self, int signal, int pid)

{

	struct action *action = actions_new(self);



	self->present[ACTION_SIGNAL] = true;

	action->type = ACTION_SIGNAL;

	action->signal = signal;

	action->pid = pid;



	return 0;

}



/*

 * actions_add_shell - add an action to execute a shell command

 */

int

actions_add_shell(struct actions *self, const char *command)

{

	struct action *action = actions_new(self);



	self->present[ACTION_SHELL] = true;

	action->type = ACTION_SHELL;

	action->command = calloc(strlen(command) + 1, sizeof(char));

	if (!action->command)

		return -1;

	strcpy(action->command, command);



	return 0;

}



/*

 * actions_add_continue - add an action to resume measurement

 */

int

actions_add_continue(struct actions *self)

{

	struct action *action = actions_new(self);



	self->present[ACTION_CONTINUE] = true;

	action->type = ACTION_CONTINUE;



	return 0;

}



/*

 * actions_parse - add an action based on text specification

 */

int

actions_parse(struct actions *self, const char *trigger)

{

	enum action_type type = ACTION_NONE;

	char *token;

	char trigger_c[strlen(trigger)];



	/* For ACTION_SIGNAL */

	int signal = 0, pid = 0;



	/* For ACTION_TRACE_OUTPUT */

	char *trace_output;



	strcpy(trigger_c, trigger);

	token = strtok(trigger_c, ",");



	if (strcmp(token, "trace") == 0)

		type = ACTION_TRACE_OUTPUT;

	else if (strcmp(token, "signal") == 0)

		type = ACTION_SIGNAL;

	else if (strcmp(token, "shell") == 0)

		type = ACTION_SHELL;

	else if (strcmp(token, "continue") == 0)

		type = ACTION_CONTINUE;

	else

		/* Invalid trigger type */

		return -1;



	token = strtok(NULL, ",");



	switch (type) {

	case ACTION_TRACE_OUTPUT:

		/* Takes no argument */

		if (token == NULL)

			trace_output = "timerlat_trace.txt";

		else {

			if (strlen(token) > 5 && strncmp(token, "file=", 5) == 0) {

				trace_output = token + 5;

			} else {

				/* Invalid argument */

				return -1;

			}



			token = strtok(NULL, ",");

			if (token != NULL)

				/* Only one argument allowed */

				return -1;

		}

		return actions_add_trace_output(self, trace_output);

	case ACTION_SIGNAL:

		/* Takes two arguments, num (signal) and pid */

		while (token != NULL) {

			if (strlen(token) > 4 && strncmp(token, "num=", 4) == 0) {

				signal = atoi(token + 4);

			} else if (strlen(token) > 4 && strncmp(token, "pid=", 4) == 0) {

				if (strncmp(token + 4, "parent", 7) == 0)

					pid = -1;

				else

					pid = atoi(token + 4);

			} else {

				/* Invalid argument */

				return -1;

			}



			token = strtok(NULL, ",");

		}



		if (!signal || !pid)

			/* Missing argument */

			return -1;



		return actions_add_signal(self, signal, pid);

	case ACTION_SHELL:

		if (token == NULL)

			return -1;

		if (strlen(token) > 8 && strncmp(token, "command=", 8) == 0)

			return actions_add_shell(self, token + 8);

		return -1;

	case ACTION_CONTINUE:

		/* Takes no argument */

		if (token != NULL)

			return -1;

		return actions_add_continue(self);

	default:

		return -1;

	}

}



/*

 * actions_perform - perform all actions

 */

int

actions_perform(struct actions *self)

{

	int pid, retval;

	const struct action *action;



	for (action = self->list; action < self->list + self->len; action++) {

		switch (action->type) {

		case ACTION_TRACE_OUTPUT:

			retval = save_trace_to_file(self->trace_output_inst, action->trace_output);

			if (retval) {

				err_msg("Error saving trace\n");

				return retval;

			}

			break;

		case ACTION_SIGNAL:

			if (action->pid == -1)

				pid = getppid();

			else

				pid = action->pid;

			retval = kill(pid, action->signal);

			if (retval) {

				err_msg("Error sending signal\n");

				return retval;

			}

			break;

		case ACTION_SHELL:

			retval = system(action->command);

			if (retval)

				return retval;

			break;

		case ACTION_CONTINUE:

			self->continue_flag = true;

			return 0;

		default:

			break;

		}

	}



	return 0;

}

tools/tracing/rtla/src/actions.h

0 → 100644
+52 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

#include <tracefs.h>

#include <stdbool.h>



enum action_type {

	ACTION_NONE = 0,

	ACTION_TRACE_OUTPUT,

	ACTION_SIGNAL,

	ACTION_SHELL,

	ACTION_CONTINUE,

	ACTION_FIELD_N

};



struct action {

	enum action_type type;

	union {

		struct {

			/* For ACTION_TRACE_OUTPUT */

			char *trace_output;

		};

		struct {

			/* For ACTION_SIGNAL */

			int signal;

			int pid;

		};

		struct {

			/* For ACTION_SHELL */

			char *command;

		};

	};

};



static const int action_default_size = 8;



struct actions {

	struct action *list;

	int len, size;

	bool present[ACTION_FIELD_N];

	bool continue_flag;



	/* External dependencies */

	struct tracefs_instance *trace_output_inst;

};



void actions_init(struct actions *self);

void actions_destroy(struct actions *self);

int actions_add_trace_output(struct actions *self, const char *trace_output);

int actions_add_signal(struct actions *self, int signal, int pid);

int actions_add_shell(struct actions *self, const char *command);

int actions_add_continue(struct actions *self);

int actions_parse(struct actions *self, const char *trigger);

int actions_perform(struct actions *self);

tools/tracing/rtla/src/timerlat.bpf.c

+9 −4
Original line number Diff line number Diff line
@@ -28,6 +28,13 @@ struct { 
	__type(value, unsigned long long);

} summary_irq SEC(".maps"), summary_thread SEC(".maps"), summary_user SEC(".maps");



struct {

	__uint(type, BPF_MAP_TYPE_ARRAY);

	__uint(max_entries, 1);

	__type(key, unsigned int);

	__type(value, unsigned long long);

} stop_tracing SEC(".maps");



struct {

	__uint(type, BPF_MAP_TYPE_RINGBUF);

	__uint(max_entries, 1);
@@ -41,8 +48,6 @@ const volatile int irq_threshold; 
const volatile int thread_threshold;

const volatile bool aa_only;



int stop_tracing;



nosubprog unsigned long long map_get(void *map,

				     unsigned int key)

{
@@ -109,7 +114,7 @@ nosubprog void set_stop_tracing(void) 
	int value = 0;



	/* Suppress further sample processing */

	stop_tracing = 1;

	map_set(&stop_tracing, 0, 1);



	/* Signal to userspace */

	bpf_ringbuf_output(&signal_stop_tracing, &value, sizeof(value), 0);
@@ -121,7 +126,7 @@ int handle_timerlat_sample(struct trace_event_raw_timerlat_sample *tp_args) 
	unsigned long long latency, latency_us;

	int bucket;



	if (stop_tracing)

	if (map_get(&stop_tracing, 0))

		return 0;



	latency = tp_args->timer_latency / output_divisor;