Commit 3c00ed34 authored by Kan Liang's avatar Kan Liang Committed by Peter Zijlstra
Browse files

perf/x86/intel/ds: Factor out functions for PEBS records processing 



Factor out functions to process normal and the last PEBS records, which
can be shared with the later patch.

Move the event updating related codes (intel_pmu_save_and_restart())
to the end, where all samples have been processed.
For the current usage, it doesn't matter when perf updates event counts
and reset the counter. Because all counters are stopped when the PEBS
buffer is drained.
Drop the return of the !intel_pmu_save_and_restart(event) check. Because
it never happen. The intel_pmu_save_and_restart(event) only returns 0,
when !hwc->event_base or the period_left > 0.
- The !hwc->event_base is impossible for the PEBS event, since the PEBS
  event is only available on GP and fixed counters, which always have
  a valid hwc->event_base.
- The check only happens for the case of non-AUTO_RELOAD and single
  PEBS, which implies that the event must be overflowed. The period_left
  must be always <= 0 for an overflowed event after the
  x86_pmu_update().

Co-developed-by: default avatar"Peter Zijlstra (Intel)" <peterz@infradead.org>
Signed-off-by: default avatar"Peter Zijlstra (Intel)" <peterz@infradead.org>
Signed-off-by: default avatarKan Liang <kan.liang@linux.intel.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20241119135504.1463839-4-kan.liang@linux.intel.com
parent 7087bfb0

arch/x86/events/intel/ds.c

+67 −42
Original line number Diff line number Diff line
@@ -2164,47 +2164,34 @@ intel_pmu_save_and_restart_reload(struct perf_event *event, int count) 
	return 0;

}



typedef void (*setup_fn)(struct perf_event *, struct pt_regs *, void *,

			 struct perf_sample_data *, struct pt_regs *);



static struct pt_regs dummy_iregs;



static __always_inline void

__intel_pmu_pebs_event(struct perf_event *event,

		       struct pt_regs *iregs,

		       struct pt_regs *regs,

		       struct perf_sample_data *data,

		       void *base, void *top,

		       int bit, int count,

		       void (*setup_sample)(struct perf_event *,

					    struct pt_regs *,

					    void *,

					    struct perf_sample_data *,

					    struct pt_regs *))

		       void *at,

		       setup_fn setup_sample)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct hw_perf_event *hwc = &event->hw;

	struct x86_perf_regs perf_regs;

	struct pt_regs *regs = &perf_regs.regs;

	void *at = get_next_pebs_record_by_bit(base, top, bit);

	static struct pt_regs dummy_iregs;



	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {

		/*

		 * Now, auto-reload is only enabled in fixed period mode.

		 * The reload value is always hwc->sample_period.

		 * May need to change it, if auto-reload is enabled in

		 * freq mode later.

		 */

		intel_pmu_save_and_restart_reload(event, count);

	} else if (!intel_pmu_save_and_restart(event))

		return;



	if (!iregs)

		iregs = &dummy_iregs;



	while (count > 1) {

	setup_sample(event, iregs, at, data, regs);

	perf_event_output(event, data, regs);

		at += cpuc->pebs_record_size;

		at = get_next_pebs_record_by_bit(at, top, bit);

		count--;

}



static __always_inline void

__intel_pmu_pebs_last_event(struct perf_event *event,

			    struct pt_regs *iregs,

			    struct pt_regs *regs,

			    struct perf_sample_data *data,

			    void *at,

			    int count,

			    setup_fn setup_sample)

{

	struct hw_perf_event *hwc = &event->hw;



	setup_sample(event, iregs, at, data, regs);

	if (iregs == &dummy_iregs) {

		/*
@@ -2222,6 +2209,44 @@ __intel_pmu_pebs_event(struct perf_event *event, 
		if (perf_event_overflow(event, data, regs))

			x86_pmu_stop(event, 0);

	}



	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {

		/*

		 * Now, auto-reload is only enabled in fixed period mode.

		 * The reload value is always hwc->sample_period.

		 * May need to change it, if auto-reload is enabled in

		 * freq mode later.

		 */

		intel_pmu_save_and_restart_reload(event, count);

	} else

		intel_pmu_save_and_restart(event);

}



static __always_inline void

__intel_pmu_pebs_events(struct perf_event *event,

			struct pt_regs *iregs,

			struct perf_sample_data *data,

			void *base, void *top,

			int bit, int count,

			setup_fn setup_sample)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct x86_perf_regs perf_regs;

	struct pt_regs *regs = &perf_regs.regs;

	void *at = get_next_pebs_record_by_bit(base, top, bit);

	int cnt = count;



	if (!iregs)

		iregs = &dummy_iregs;



	while (cnt > 1) {

		__intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample);

		at += cpuc->pebs_record_size;

		at = get_next_pebs_record_by_bit(at, top, bit);

		cnt--;

	}



	__intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample);

}



static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
@@ -2258,7 +2283,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_ 
		return;

	}



	__intel_pmu_pebs_event(event, iregs, data, at, top, 0, n,

	__intel_pmu_pebs_events(event, iregs, data, at, top, 0, n,

				setup_pebs_fixed_sample_data);

}
@@ -2390,7 +2415,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d 
		}



		if (counts[bit]) {

			__intel_pmu_pebs_event(event, iregs, data, base,

			__intel_pmu_pebs_events(event, iregs, data, base,

						top, bit, counts[bit],

						setup_pebs_fixed_sample_data);

		}
@@ -2444,7 +2469,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d 
		if (WARN_ON_ONCE(!event->attr.precise_ip))

			continue;



		__intel_pmu_pebs_event(event, iregs, data, base,

		__intel_pmu_pebs_events(event, iregs, data, base,

					top, bit, counts[bit],

					setup_pebs_adaptive_sample_data);

	}