Commit 0259bf63 authored by Namhyung Kim's avatar Namhyung Kim Committed by Ingo Molnar
Browse files

perf/core: Optimize perf_adjust_freq_unthr_context() 



It was unnecessarily disabling and enabling PMUs for each event.  It
should be done at PMU level.  Add pmu_ctx->nr_freq counter to check it
at each PMU.  As PMU context has separate active lists for pinned group
and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT
even if it needs to unthrottle sampling events.

Signed-off-by: default avatarNamhyung Kim <namhyung@kernel.org>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Tested-by: default avatarMingwei Zhang <mizhang@google.com>
Reviewed-by: default avatarIan Rogers <irogers@google.com>
Reviewed-by: default avatarKan Liang <kan.liang@linux.intel.com>
Link: https://lore.kernel.org/r/20240207050545.2727923-1-namhyung@kernel.org
parent 9794563d

include/linux/perf_event.h

+6 −0
Original line number Diff line number Diff line
@@ -883,6 +883,7 @@ struct perf_event_pmu_context { 


	unsigned int			nr_events;

	unsigned int			nr_cgroups;

	unsigned int			nr_freq;



	atomic_t			refcount; /* event <-> epc */

	struct rcu_head			rcu_head;
@@ -897,6 +898,11 @@ struct perf_event_pmu_context { 
	int				rotate_necessary;

};



static inline bool perf_pmu_ctx_is_active(struct perf_event_pmu_context *epc)

{

	return !list_empty(&epc->flexible_active) || !list_empty(&epc->pinned_active);

}



struct perf_event_groups {

	struct rb_root	tree;

	u64		index;

kernel/events/core.c

+44 −26
Original line number Diff line number Diff line
@@ -2302,8 +2302,10 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx) 


	if (!is_software_event(event))

		cpc->active_oncpu--;

	if (event->attr.freq && event->attr.sample_freq)

	if (event->attr.freq && event->attr.sample_freq) {

		ctx->nr_freq--;

		epc->nr_freq--;

	}

	if (event->attr.exclusive || !cpc->active_oncpu)

		cpc->exclusive = 0;
@@ -2558,9 +2560,10 @@ event_sched_in(struct perf_event *event, struct perf_event_context *ctx) 


	if (!is_software_event(event))

		cpc->active_oncpu++;

	if (event->attr.freq && event->attr.sample_freq)

	if (event->attr.freq && event->attr.sample_freq) {

		ctx->nr_freq++;



		epc->nr_freq++;

	}

	if (event->attr.exclusive)

		cpc->exclusive = 1;
@@ -4123,30 +4126,14 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bo 
	}

}



/*

 * combine freq adjustment with unthrottling to avoid two passes over the

 * events. At the same time, make sure, having freq events does not change

 * the rate of unthrottling as that would introduce bias.

 */

static void

perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)

static void perf_adjust_freq_unthr_events(struct list_head *event_list)

{

	struct perf_event *event;

	struct hw_perf_event *hwc;

	u64 now, period = TICK_NSEC;

	s64 delta;



	/*

	 * only need to iterate over all events iff:

	 * - context have events in frequency mode (needs freq adjust)

	 * - there are events to unthrottle on this cpu

	 */

	if (!(ctx->nr_freq || unthrottle))

		return;



	raw_spin_lock(&ctx->lock);



	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {

	list_for_each_entry(event, event_list, active_list) {

		if (event->state != PERF_EVENT_STATE_ACTIVE)

			continue;
@@ -4154,8 +4141,6 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle) 
		if (!event_filter_match(event))

			continue;



		perf_pmu_disable(event->pmu);



		hwc = &event->hw;



		if (hwc->interrupts == MAX_INTERRUPTS) {
@@ -4165,7 +4150,7 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle) 
		}



		if (!event->attr.freq || !event->attr.sample_freq)

			goto next;

			continue;



		/*

		 * stop the event and update event->count
@@ -4187,8 +4172,41 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle) 
			perf_adjust_period(event, period, delta, false);



		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);

	next:

		perf_pmu_enable(event->pmu);

	}

}



/*

 * combine freq adjustment with unthrottling to avoid two passes over the

 * events. At the same time, make sure, having freq events does not change

 * the rate of unthrottling as that would introduce bias.

 */

static void

perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)

{

	struct perf_event_pmu_context *pmu_ctx;



	/*

	 * only need to iterate over all events iff:

	 * - context have events in frequency mode (needs freq adjust)

	 * - there are events to unthrottle on this cpu

	 */

	if (!(ctx->nr_freq || unthrottle))

		return;



	raw_spin_lock(&ctx->lock);



	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {

		if (!(pmu_ctx->nr_freq || unthrottle))

			continue;

		if (!perf_pmu_ctx_is_active(pmu_ctx))

			continue;

		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)

			continue;



		perf_pmu_disable(pmu_ctx->pmu);

		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);

		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);

		perf_pmu_enable(pmu_ctx->pmu);

	}



	raw_spin_unlock(&ctx->lock);