Commit d243d0bb authored by Dapeng Mi's avatar Dapeng Mi Committed by Peter Zijlstra
Browse files

perf/x86/intel: Initialize architectural PEBS 



arch-PEBS leverages CPUID.23H.4/5 sub-leaves enumerate arch-PEBS
supported capabilities and counters bitmap. This patch parses these 2
sub-leaves and initializes arch-PEBS capabilities and corresponding
structures.

Since IA32_PEBS_ENABLE and MSR_PEBS_DATA_CFG MSRs are no longer existed
for arch-PEBS, arch-PEBS doesn't need to manipulate these MSRs. Thus add
a simple pair of __intel_pmu_pebs_enable/disable() callbacks for
arch-PEBS.

Signed-off-by: default avatarDapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251029102136.61364-6-dapeng1.mi@linux.intel.com
parent 5e4e355a

arch/x86/events/core.c

+16 −5
Original line number Diff line number Diff line
@@ -554,14 +554,22 @@ static inline int precise_br_compat(struct perf_event *event) 
	return m == b;

}



int x86_pmu_max_precise(void)

int x86_pmu_max_precise(struct pmu *pmu)

{

	int precise = 0;



	/* Support for constant skid */

	if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {

		/* arch PEBS */

		if (x86_pmu.arch_pebs) {

			precise = 2;

			if (hybrid(pmu, arch_pebs_cap).pdists)

				precise++;



			return precise;

		}



		/* legacy PEBS - support for constant skid */

		precise++;

		/* Support for IP fixup */

		if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)

			precise++;
@@ -569,13 +577,14 @@ int x86_pmu_max_precise(void) 
		if (x86_pmu.pebs_prec_dist)

			precise++;

	}



	return precise;

}



int x86_pmu_hw_config(struct perf_event *event)

{

	if (event->attr.precise_ip) {

		int precise = x86_pmu_max_precise();

		int precise = x86_pmu_max_precise(event->pmu);



		if (event->attr.precise_ip > precise)

			return -EOPNOTSUPP;
@@ -2630,7 +2639,9 @@ static ssize_t max_precise_show(struct device *cdev, 
				  struct device_attribute *attr,

				  char *buf)

{

	return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());

	struct pmu *pmu = dev_get_drvdata(cdev);



	return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise(pmu));

}



static DEVICE_ATTR_RO(max_precise);

arch/x86/events/intel/core.c

+44 −16
Original line number Diff line number Diff line
@@ -5271,34 +5271,59 @@ static inline bool intel_pmu_broken_perf_cap(void) 
	return false;

}



#define counter_mask(_gp, _fixed) ((_gp) | ((u64)(_fixed) << INTEL_PMC_IDX_FIXED))



static void update_pmu_cap(struct pmu *pmu)

{

	unsigned int cntr, fixed_cntr, ecx, edx;

	union cpuid35_eax eax;

	union cpuid35_ebx ebx;

	unsigned int eax, ebx, ecx, edx;

	union cpuid35_eax eax_0;

	union cpuid35_ebx ebx_0;

	u64 cntrs_mask = 0;

	u64 pebs_mask = 0;

	u64 pdists_mask = 0;



	cpuid(ARCH_PERFMON_EXT_LEAF, &eax.full, &ebx.full, &ecx, &edx);

	cpuid(ARCH_PERFMON_EXT_LEAF, &eax_0.full, &ebx_0.full, &ecx, &edx);



	if (ebx.split.umask2)

	if (ebx_0.split.umask2)

		hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_UMASK2;

	if (ebx.split.eq)

	if (ebx_0.split.eq)

		hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_EQ;



	if (eax.split.cntr_subleaf) {

	if (eax_0.split.cntr_subleaf) {

		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,

			    &cntr, &fixed_cntr, &ecx, &edx);

		hybrid(pmu, cntr_mask64) = cntr;

		hybrid(pmu, fixed_cntr_mask64) = fixed_cntr;

			    &eax, &ebx, &ecx, &edx);

		hybrid(pmu, cntr_mask64) = eax;

		hybrid(pmu, fixed_cntr_mask64) = ebx;

		cntrs_mask = counter_mask(eax, ebx);

	}



	if (eax.split.acr_subleaf) {

	if (eax_0.split.acr_subleaf) {

		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_ACR_LEAF,

			    &cntr, &fixed_cntr, &ecx, &edx);

			    &eax, &ebx, &ecx, &edx);

		/* The mask of the counters which can be reloaded */

		hybrid(pmu, acr_cntr_mask64) = cntr | ((u64)fixed_cntr << INTEL_PMC_IDX_FIXED);



		hybrid(pmu, acr_cntr_mask64) = counter_mask(eax, ebx);

		/* The mask of the counters which can cause a reload of reloadable counters */

		hybrid(pmu, acr_cause_mask64) = ecx | ((u64)edx << INTEL_PMC_IDX_FIXED);

		hybrid(pmu, acr_cause_mask64) = counter_mask(ecx, edx);

	}



	/* Bits[5:4] should be set simultaneously if arch-PEBS is supported */

	if (eax_0.split.pebs_caps_subleaf && eax_0.split.pebs_cnts_subleaf) {

		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_CAP_LEAF,

			    &eax, &ebx, &ecx, &edx);

		hybrid(pmu, arch_pebs_cap).caps = (u64)ebx << 32;



		cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_COUNTER_LEAF,

			    &eax, &ebx, &ecx, &edx);

		pebs_mask   = counter_mask(eax, ecx);

		pdists_mask = counter_mask(ebx, edx);

		hybrid(pmu, arch_pebs_cap).counters = pebs_mask;

		hybrid(pmu, arch_pebs_cap).pdists = pdists_mask;



		if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask))

			x86_pmu.arch_pebs = 0;

	} else {

		WARN_ON(x86_pmu.arch_pebs == 1);

		x86_pmu.arch_pebs = 0;

	}



	if (!intel_pmu_broken_perf_cap()) {
@@ -6252,7 +6277,7 @@ tsx_is_visible(struct kobject *kobj, struct attribute *attr, int i) 
static umode_t

pebs_is_visible(struct kobject *kobj, struct attribute *attr, int i)

{

	return x86_pmu.ds_pebs ? attr->mode : 0;

	return intel_pmu_has_pebs() ? attr->mode : 0;

}



static umode_t
@@ -7728,6 +7753,9 @@ __init int intel_pmu_init(void) 
	if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))

		update_pmu_cap(NULL);



	if (x86_pmu.arch_pebs)

		pr_cont("Architectural PEBS, ");



	intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,

				      &x86_pmu.fixed_cntr_mask64,

				      &x86_pmu.intel_ctrl);

arch/x86/events/intel/ds.c

+44 −8
Original line number Diff line number Diff line
@@ -1531,6 +1531,15 @@ static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc) 
		intel_pmu_drain_pebs_buffer();

}



static void __intel_pmu_pebs_enable(struct perf_event *event)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct hw_perf_event *hwc = &event->hw;



	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;

	cpuc->pebs_enabled |= 1ULL << hwc->idx;

}



void intel_pmu_pebs_enable(struct perf_event *event)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@@ -1539,9 +1548,7 @@ void intel_pmu_pebs_enable(struct perf_event *event) 
	struct debug_store *ds = cpuc->ds;

	unsigned int idx = hwc->idx;



	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;



	cpuc->pebs_enabled |= 1ULL << hwc->idx;

	__intel_pmu_pebs_enable(event);



	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))

		cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
@@ -1603,14 +1610,22 @@ void intel_pmu_pebs_del(struct perf_event *event) 
	pebs_update_state(needed_cb, cpuc, event, false);

}



void intel_pmu_pebs_disable(struct perf_event *event)

static void __intel_pmu_pebs_disable(struct perf_event *event)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct hw_perf_event *hwc = &event->hw;



	intel_pmu_drain_large_pebs(cpuc);



	cpuc->pebs_enabled &= ~(1ULL << hwc->idx);

	hwc->config |= ARCH_PERFMON_EVENTSEL_INT;

}



void intel_pmu_pebs_disable(struct perf_event *event)

{

	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	struct hw_perf_event *hwc = &event->hw;



	__intel_pmu_pebs_disable(event);



	if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&

	    (x86_pmu.version < 5))
@@ -1622,8 +1637,6 @@ void intel_pmu_pebs_disable(struct perf_event *event) 


	if (cpuc->enabled)

		wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);



	hwc->config |= ARCH_PERFMON_EVENTSEL_INT;

}



void intel_pmu_pebs_enable_all(void)
@@ -2669,11 +2682,26 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d 
	}

}



static void __init intel_arch_pebs_init(void)

{

	/*

	 * Current hybrid platforms always both support arch-PEBS or not

	 * on all kinds of cores. So directly set x86_pmu.arch_pebs flag

	 * if boot cpu supports arch-PEBS.

	 */

	x86_pmu.arch_pebs = 1;

	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;

	x86_pmu.pebs_capable = ~0ULL;



	x86_pmu.pebs_enable = __intel_pmu_pebs_enable;

	x86_pmu.pebs_disable = __intel_pmu_pebs_disable;

}



/*

 * PEBS probe and setup

 */



void __init intel_pebs_init(void)

static void __init intel_ds_pebs_init(void)

{

	/*

	 * No support for 32bit formats
@@ -2788,6 +2816,14 @@ void __init intel_pebs_init(void) 
	}

}



void __init intel_pebs_init(void)

{

	if (x86_pmu.intel_cap.pebs_format == 0xf)

		intel_arch_pebs_init();

	else

		intel_ds_pebs_init();

}



void perf_restore_debug_store(void)

{

	struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);

arch/x86/events/perf_event.h

+22 −3
Original line number Diff line number Diff line
@@ -708,6 +708,12 @@ enum hybrid_pmu_type { 
	hybrid_big_small_tiny	= hybrid_big   | hybrid_small_tiny,

};



struct arch_pebs_cap {

	u64 caps;

	u64 counters;

	u64 pdists;

};



struct x86_hybrid_pmu {

	struct pmu			pmu;

	const char			*name;
@@ -752,6 +758,8 @@ struct x86_hybrid_pmu { 
					mid_ack		:1,

					enabled_ack	:1;



	struct arch_pebs_cap		arch_pebs_cap;



	u64				pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];

};
@@ -906,7 +914,7 @@ struct x86_pmu { 
	union perf_capabilities intel_cap;



	/*

	 * Intel DebugStore bits

	 * Intel DebugStore and PEBS bits

	 */

	unsigned int	bts			:1,

			bts_active		:1,
@@ -917,7 +925,8 @@ struct x86_pmu { 
			pebs_no_tlb		:1,

			pebs_no_isolation	:1,

			pebs_block		:1,

			pebs_ept		:1;

			pebs_ept		:1,

			arch_pebs		:1;

	int		pebs_record_size;

	int		pebs_buffer_size;

	u64		pebs_events_mask;
@@ -929,6 +938,11 @@ struct x86_pmu { 
	u64		rtm_abort_event;

	u64		pebs_capable;



	/*

	 * Intel Architectural PEBS

	 */

	struct arch_pebs_cap arch_pebs_cap;



	/*

	 * Intel LBR

	 */
@@ -1216,7 +1230,7 @@ int x86_reserve_hardware(void); 


void x86_release_hardware(void);



int x86_pmu_max_precise(void);

int x86_pmu_max_precise(struct pmu *pmu);



void hw_perf_lbr_event_destroy(struct perf_event *event);
@@ -1791,6 +1805,11 @@ static inline int intel_pmu_max_num_pebs(struct pmu *pmu) 
	return fls((u32)hybrid(pmu, pebs_events_mask));

}



static inline bool intel_pmu_has_pebs(void)

{

	return x86_pmu.ds_pebs || x86_pmu.arch_pebs;

}



#else /* CONFIG_CPU_SUP_INTEL */



static inline void reserve_ds_buffers(void)

arch/x86/include/asm/perf_event.h

+6 −1
Original line number Diff line number Diff line
@@ -200,6 +200,8 @@ union cpuid10_edx { 
#define ARCH_PERFMON_EXT_LEAF			0x00000023

#define ARCH_PERFMON_NUM_COUNTER_LEAF		0x1

#define ARCH_PERFMON_ACR_LEAF			0x2

#define ARCH_PERFMON_PEBS_CAP_LEAF		0x4

#define ARCH_PERFMON_PEBS_COUNTER_LEAF		0x5



union cpuid35_eax {

	struct {
@@ -210,7 +212,10 @@ union cpuid35_eax { 
		unsigned int    acr_subleaf:1;

		/* Events Sub-Leaf */

		unsigned int    events_subleaf:1;

		unsigned int	reserved:28;

		/* arch-PEBS Sub-Leaves */

		unsigned int	pebs_caps_subleaf:1;

		unsigned int	pebs_cnts_subleaf:1;

		unsigned int	reserved:26;

	} split;

	unsigned int            full;

};