Commit 576a997c authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'perf-core-2024-07-16' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull performance events updates from Ingo Molnar:

 - Intel PT support enhancements & fixes

 - Fix leaked SIGTRAP events

 - Improve and fix the Intel uncore driver

 - Add support for Intel HBM and CXL uncore counters

 - Add Intel Lake and Arrow Lake support

 - AMD uncore driver fixes

 - Make SIGTRAP and __perf_pending_irq() work on RT

 - Micro-optimizations

 - Misc cleanups and fixes

* tag 'perf-core-2024-07-16' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (44 commits)
  perf/x86/intel: Add a distinct name for Granite Rapids
  perf/x86/intel/ds: Fix non 0 retire latency on Raptorlake
  perf/x86/intel: Hide Topdown metrics events if the feature is not enumerated
  perf/x86/intel/uncore: Fix the bits of the CHA extended umask for SPR
  perf: Split __perf_pending_irq() out of perf_pending_irq()
  perf: Don't disable preemption in perf_pending_task().
  perf: Move swevent_htable::recursion into task_struct.
  perf: Shrink the size of the recursion counter.
  perf: Enqueue SIGTRAP always via task_work.
  task_work: Add TWA_NMI_CURRENT as an additional notify mode.
  perf: Move irq_work_queue() where the event is prepared.
  perf: Fix event leak upon exec and file release
  perf: Fix event leak upon exit
  task_work: Introduce task_work_cancel() again
  task_work: s/task_work_cancel()/task_work_cancel_func()/
  perf/x86/amd/uncore: Fix DF and UMC domain identification
  perf/x86/amd/uncore: Avoid PMU registration if counters are unavailable
  perf/x86/intel: Support Perfmon MSRs aliasing
  perf/x86/intel: Support PERFEVTSEL extension
  perf/x86: Add config_mask to represent EVENTSEL bitmask
  ...
parents 4a996d90 fa0c1c9d

arch/x86/events/amd/core.c

+15 −13
Original line number Diff line number Diff line
@@ -432,8 +432,10 @@ static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc, 
	 * be removed on one CPU at a time AND PMU is disabled

	 * when we come here

	 */

	for (i = 0; i < x86_pmu.num_counters; i++) {

		if (cmpxchg(nb->owners + i, event, NULL) == event)

	for_each_set_bit(i, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		struct perf_event *tmp = event;



		if (try_cmpxchg(nb->owners + i, &tmp, NULL))

			break;

	}

}
@@ -499,7 +501,7 @@ __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *ev 
	 * because of successive calls to x86_schedule_events() from

	 * hw_perf_group_sched_in() without hw_perf_enable()

	 */

	for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {

	for_each_set_bit(idx, c->idxmsk, x86_pmu_max_num_counters(NULL)) {

		if (new == -1 || hwc->idx == idx)

			/* assign free slot, prefer hwc->idx */

			old = cmpxchg(nb->owners + idx, NULL, event);
@@ -542,7 +544,7 @@ static struct amd_nb *amd_alloc_nb(int cpu) 
	/*

	 * initialize all possible NB constraints

	 */

	for (i = 0; i < x86_pmu.num_counters; i++) {

	for_each_set_bit(i, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		__set_bit(i, nb->event_constraints[i].idxmsk);

		nb->event_constraints[i].weight = 1;

	}
@@ -735,7 +737,7 @@ static void amd_pmu_check_overflow(void) 
	 * counters are always enabled when this function is called and

	 * ARCH_PERFMON_EVENTSEL_INT is always set.

	 */

	for (idx = 0; idx < x86_pmu.num_counters; idx++) {

	for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		if (!test_bit(idx, cpuc->active_mask))

			continue;
@@ -755,7 +757,7 @@ static void amd_pmu_enable_all(int added) 


	amd_brs_enable_all();



	for (idx = 0; idx < x86_pmu.num_counters; idx++) {

	for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		/* only activate events which are marked as active */

		if (!test_bit(idx, cpuc->active_mask))

			continue;
@@ -978,7 +980,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) 
	/* Clear any reserved bits set by buggy microcode */

	status &= amd_pmu_global_cntr_mask;



	for (idx = 0; idx < x86_pmu.num_counters; idx++) {

	for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		if (!test_bit(idx, cpuc->active_mask))

			continue;
@@ -1313,7 +1315,7 @@ static __initconst const struct x86_pmu amd_pmu = { 
	.addr_offset            = amd_pmu_addr_offset,

	.event_map		= amd_pmu_event_map,

	.max_events		= ARRAY_SIZE(amd_perfmon_event_map),

	.num_counters		= AMD64_NUM_COUNTERS,

	.cntr_mask64		= GENMASK_ULL(AMD64_NUM_COUNTERS - 1, 0),

	.add			= amd_pmu_add_event,

	.del			= amd_pmu_del_event,

	.cntval_bits		= 48,
@@ -1412,7 +1414,7 @@ static int __init amd_core_pmu_init(void) 
	 */

	x86_pmu.eventsel	= MSR_F15H_PERF_CTL;

	x86_pmu.perfctr		= MSR_F15H_PERF_CTR;

	x86_pmu.num_counters	= AMD64_NUM_COUNTERS_CORE;

	x86_pmu.cntr_mask64	= GENMASK_ULL(AMD64_NUM_COUNTERS_CORE - 1, 0);



	/* Check for Performance Monitoring v2 support */

	if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) {
@@ -1422,9 +1424,9 @@ static int __init amd_core_pmu_init(void) 
		x86_pmu.version = 2;



		/* Find the number of available Core PMCs */

		x86_pmu.num_counters = ebx.split.num_core_pmc;

		x86_pmu.cntr_mask64 = GENMASK_ULL(ebx.split.num_core_pmc - 1, 0);



		amd_pmu_global_cntr_mask = (1ULL << x86_pmu.num_counters) - 1;

		amd_pmu_global_cntr_mask = x86_pmu.cntr_mask64;



		/* Update PMC handling functions */

		x86_pmu.enable_all = amd_pmu_v2_enable_all;
@@ -1452,12 +1454,12 @@ static int __init amd_core_pmu_init(void) 
		 * even numbered counter that has a consecutive adjacent odd

		 * numbered counter following it.

		 */

		for (i = 0; i < x86_pmu.num_counters - 1; i += 2)

		for (i = 0; i < x86_pmu_max_num_counters(NULL) - 1; i += 2)

			even_ctr_mask |= BIT_ULL(i);



		pair_constraint = (struct event_constraint)

				    __EVENT_CONSTRAINT(0, even_ctr_mask, 0,

				    x86_pmu.num_counters / 2, 0,

				    x86_pmu_max_num_counters(NULL) / 2, 0,

				    PERF_X86_EVENT_PAIR);



		x86_pmu.get_event_constraints = amd_get_event_constraints_f17h;

arch/x86/events/amd/uncore.c

+23 −13
Original line number Diff line number Diff line
@@ -162,7 +162,9 @@ static int amd_uncore_add(struct perf_event *event, int flags) 
	/* if not, take the first available counter */

	hwc->idx = -1;

	for (i = 0; i < pmu->num_counters; i++) {

		if (cmpxchg(&ctx->events[i], NULL, event) == NULL) {

		struct perf_event *tmp = NULL;



		if (try_cmpxchg(&ctx->events[i], &tmp, event)) {

			hwc->idx = i;

			break;

		}
@@ -196,7 +198,9 @@ static void amd_uncore_del(struct perf_event *event, int flags) 
	event->pmu->stop(event, PERF_EF_UPDATE);



	for (i = 0; i < pmu->num_counters; i++) {

		if (cmpxchg(&ctx->events[i], event, NULL) == event)

		struct perf_event *tmp = event;



		if (try_cmpxchg(&ctx->events[i], &tmp, NULL))

			break;

	}
@@ -639,7 +643,7 @@ void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) 
	info.split.aux_data = 0;

	info.split.num_pmcs = NUM_COUNTERS_NB;

	info.split.gid = 0;

	info.split.cid = topology_die_id(cpu);

	info.split.cid = topology_logical_package_id(cpu);



	if (pmu_version >= 2) {

		ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
@@ -654,17 +658,20 @@ int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 
{

	struct attribute **df_attr = amd_uncore_df_format_attr;

	struct amd_uncore_pmu *pmu;

	int num_counters;



	/* Run just once */

	if (uncore->init_done)

		return amd_uncore_ctx_init(uncore, cpu);



	num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);

	if (!num_counters)

		goto done;



	/* No grouping, single instance for a system */

	uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);

	if (!uncore->pmus) {

		uncore->num_pmus = 0;

	if (!uncore->pmus)

		goto done;

	}



	/*

	 * For Family 17h and above, the Northbridge counters are repurposed
@@ -674,7 +681,7 @@ int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 
	pmu = &uncore->pmus[0];

	strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_df" : "amd_nb",

		sizeof(pmu->name));

	pmu->num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);

	pmu->num_counters = num_counters;

	pmu->msr_base = MSR_F15H_NB_PERF_CTL;

	pmu->rdpmc_base = RDPMC_BASE_NB;

	pmu->group = amd_uncore_ctx_gid(uncore, cpu);
@@ -785,17 +792,20 @@ int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 
{

	struct attribute **l3_attr = amd_uncore_l3_format_attr;

	struct amd_uncore_pmu *pmu;

	int num_counters;



	/* Run just once */

	if (uncore->init_done)

		return amd_uncore_ctx_init(uncore, cpu);



	num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);

	if (!num_counters)

		goto done;



	/* No grouping, single instance for a system */

	uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);

	if (!uncore->pmus) {

		uncore->num_pmus = 0;

	if (!uncore->pmus)

		goto done;

	}



	/*

	 * For Family 17h and above, L3 cache counters are available instead
@@ -805,7 +815,7 @@ int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu) 
	pmu = &uncore->pmus[0];

	strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_l3" : "amd_l2",

		sizeof(pmu->name));

	pmu->num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);

	pmu->num_counters = num_counters;

	pmu->msr_base = MSR_F16H_L2I_PERF_CTL;

	pmu->rdpmc_base = RDPMC_BASE_LLC;

	pmu->group = amd_uncore_ctx_gid(uncore, cpu);
@@ -893,8 +903,8 @@ void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) 
	cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx);

	info.split.aux_data = ecx;	/* stash active mask */

	info.split.num_pmcs = ebx.split.num_umc_pmc;

	info.split.gid = topology_die_id(cpu);

	info.split.cid = topology_die_id(cpu);

	info.split.gid = topology_logical_package_id(cpu);

	info.split.cid = topology_logical_package_id(cpu);

	*per_cpu_ptr(uncore->info, cpu) = info;

}

arch/x86/events/core.c

+54 −56
Original line number Diff line number Diff line
@@ -189,29 +189,31 @@ static DEFINE_MUTEX(pmc_reserve_mutex); 


#ifdef CONFIG_X86_LOCAL_APIC



static inline int get_possible_num_counters(void)

static inline u64 get_possible_counter_mask(void)

{

	int i, num_counters = x86_pmu.num_counters;

	u64 cntr_mask = x86_pmu.cntr_mask64;

	int i;



	if (!is_hybrid())

		return num_counters;

		return cntr_mask;



	for (i = 0; i < x86_pmu.num_hybrid_pmus; i++)

		num_counters = max_t(int, num_counters, x86_pmu.hybrid_pmu[i].num_counters);

		cntr_mask |= x86_pmu.hybrid_pmu[i].cntr_mask64;



	return num_counters;

	return cntr_mask;

}



static bool reserve_pmc_hardware(void)

{

	int i, num_counters = get_possible_num_counters();

	u64 cntr_mask = get_possible_counter_mask();

	int i, end;



	for (i = 0; i < num_counters; i++) {

	for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) {

		if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))

			goto perfctr_fail;

	}



	for (i = 0; i < num_counters; i++) {

	for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) {

		if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))

			goto eventsel_fail;

	}
@@ -219,13 +221,14 @@ static bool reserve_pmc_hardware(void) 
	return true;



eventsel_fail:

	for (i--; i >= 0; i--)

	end = i;

	for_each_set_bit(i, (unsigned long *)&cntr_mask, end)

		release_evntsel_nmi(x86_pmu_config_addr(i));



	i = num_counters;

	i = X86_PMC_IDX_MAX;



perfctr_fail:

	for (i--; i >= 0; i--)

	end = i;

	for_each_set_bit(i, (unsigned long *)&cntr_mask, end)

		release_perfctr_nmi(x86_pmu_event_addr(i));



	return false;
@@ -233,9 +236,10 @@ static bool reserve_pmc_hardware(void) 


static void release_pmc_hardware(void)

{

	int i, num_counters = get_possible_num_counters();

	u64 cntr_mask = get_possible_counter_mask();

	int i;



	for (i = 0; i < num_counters; i++) {

	for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) {

		release_perfctr_nmi(x86_pmu_event_addr(i));

		release_evntsel_nmi(x86_pmu_config_addr(i));

	}
@@ -248,7 +252,8 @@ static void release_pmc_hardware(void) {} 


#endif



bool check_hw_exists(struct pmu *pmu, int num_counters, int num_counters_fixed)

bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,

		     unsigned long *fixed_cntr_mask)

{

	u64 val, val_fail = -1, val_new= ~0;

	int i, reg, reg_fail = -1, ret = 0;
@@ -259,7 +264,7 @@ bool check_hw_exists(struct pmu *pmu, int num_counters, int num_counters_fixed) 
	 * Check to see if the BIOS enabled any of the counters, if so

	 * complain and bail.

	 */

	for (i = 0; i < num_counters; i++) {

	for_each_set_bit(i, cntr_mask, X86_PMC_IDX_MAX) {

		reg = x86_pmu_config_addr(i);

		ret = rdmsrl_safe(reg, &val);

		if (ret)
@@ -273,12 +278,12 @@ bool check_hw_exists(struct pmu *pmu, int num_counters, int num_counters_fixed) 
		}

	}



	if (num_counters_fixed) {

	if (*(u64 *)fixed_cntr_mask) {

		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;

		ret = rdmsrl_safe(reg, &val);

		if (ret)

			goto msr_fail;

		for (i = 0; i < num_counters_fixed; i++) {

		for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) {

			if (fixed_counter_disabled(i, pmu))

				continue;

			if (val & (0x03ULL << i*4)) {
@@ -619,7 +624,7 @@ int x86_pmu_hw_config(struct perf_event *event) 
		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;



	if (event->attr.type == event->pmu->type)

		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;

		event->hw.config |= x86_pmu_get_event_config(event);



	if (event->attr.sample_period && x86_pmu.limit_period) {

		s64 left = event->attr.sample_period;
@@ -679,7 +684,7 @@ void x86_pmu_disable_all(void) 
	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	int idx;



	for (idx = 0; idx < x86_pmu.num_counters; idx++) {

	for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;

		u64 val;
@@ -736,7 +741,7 @@ void x86_pmu_enable_all(int added) 
	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

	int idx;



	for (idx = 0; idx < x86_pmu.num_counters; idx++) {

	for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;



		if (!test_bit(idx, cpuc->active_mask))
@@ -975,7 +980,6 @@ EXPORT_SYMBOL_GPL(perf_assign_events); 


int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)

{

	int num_counters = hybrid(cpuc->pmu, num_counters);

	struct event_constraint *c;

	struct perf_event *e;

	int n0, i, wmin, wmax, unsched = 0;
@@ -1051,7 +1055,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 


	/* slow path */

	if (i != n) {

		int gpmax = num_counters;

		int gpmax = x86_pmu_max_num_counters(cpuc->pmu);



		/*

		 * Do not allow scheduling of more than half the available
@@ -1072,7 +1076,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 
		 * the extra Merge events needed by large increment events.

		 */

		if (x86_pmu.flags & PMU_FL_PAIR) {

			gpmax = num_counters - cpuc->n_pair;

			gpmax -= cpuc->n_pair;

			WARN_ON(gpmax <= 0);

		}
@@ -1157,12 +1161,10 @@ static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event, 
 */

static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)

{

	int num_counters = hybrid(cpuc->pmu, num_counters);

	int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);

	struct perf_event *event;

	int n, max_count;



	max_count = num_counters + num_counters_fixed;

	max_count = x86_pmu_num_counters(cpuc->pmu) + x86_pmu_num_counters_fixed(cpuc->pmu);



	/* current number of events already accepted */

	n = cpuc->n_events;
@@ -1234,8 +1236,7 @@ static inline void x86_assign_hw_event(struct perf_event *event, 
		fallthrough;

	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1:

		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;

		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 +

				(idx - INTEL_PMC_IDX_FIXED);

		hwc->event_base = x86_pmu_fixed_ctr_addr(idx - INTEL_PMC_IDX_FIXED);

		hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) |

					INTEL_PMC_FIXED_RDPMC_BASE;

		break;
@@ -1522,13 +1523,13 @@ void perf_event_print_debug(void) 
	u64 pebs, debugctl;

	int cpu = smp_processor_id();

	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);

	int num_counters = hybrid(cpuc->pmu, num_counters);

	int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);

	unsigned long *cntr_mask = hybrid(cpuc->pmu, cntr_mask);

	unsigned long *fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask);

	struct event_constraint *pebs_constraints = hybrid(cpuc->pmu, pebs_constraints);

	unsigned long flags;

	int idx;



	if (!num_counters)

	if (!*(u64 *)cntr_mask)

		return;



	local_irq_save(flags);
@@ -1555,7 +1556,7 @@ void perf_event_print_debug(void) 
	}

	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);



	for (idx = 0; idx < num_counters; idx++) {

	for_each_set_bit(idx, cntr_mask, X86_PMC_IDX_MAX) {

		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);

		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
@@ -1568,10 +1569,10 @@ void perf_event_print_debug(void) 
		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",

			cpu, idx, prev_left);

	}

	for (idx = 0; idx < num_counters_fixed; idx++) {

	for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) {

		if (fixed_counter_disabled(idx, cpuc->pmu))

			continue;

		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

		rdmsrl(x86_pmu_fixed_ctr_addr(idx), pmc_count);



		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",

			cpu, idx, pmc_count);
@@ -1682,7 +1683,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) 
	 */

	apic_write(APIC_LVTPC, APIC_DM_NMI);



	for (idx = 0; idx < x86_pmu.num_counters; idx++) {

	for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {

		if (!test_bit(idx, cpuc->active_mask))

			continue;
@@ -2038,18 +2039,15 @@ static void _x86_pmu_read(struct perf_event *event) 
	static_call(x86_pmu_update)(event);

}



void x86_pmu_show_pmu_cap(int num_counters, int num_counters_fixed,

			  u64 intel_ctrl)

void x86_pmu_show_pmu_cap(struct pmu *pmu)

{

	pr_info("... version:                %d\n",     x86_pmu.version);

	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);

	pr_info("... generic registers:      %d\n",     num_counters);

	pr_info("... generic registers:      %d\n",     x86_pmu_num_counters(pmu));

	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);

	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);

	pr_info("... fixed-purpose events:   %lu\n",

			hweight64((((1ULL << num_counters_fixed) - 1)

					<< INTEL_PMC_IDX_FIXED) & intel_ctrl));

	pr_info("... event mask:             %016Lx\n", intel_ctrl);

	pr_info("... fixed-purpose events:   %d\n",     x86_pmu_num_counters_fixed(pmu));

	pr_info("... event mask:             %016Lx\n", hybrid(pmu, intel_ctrl));

}



static int __init init_hw_perf_events(void)
@@ -2086,7 +2084,7 @@ static int __init init_hw_perf_events(void) 
	pmu_check_apic();



	/* sanity check that the hardware exists or is emulated */

	if (!check_hw_exists(&pmu, x86_pmu.num_counters, x86_pmu.num_counters_fixed))

	if (!check_hw_exists(&pmu, x86_pmu.cntr_mask, x86_pmu.fixed_cntr_mask))

		goto out_bad_pmu;



	pr_cont("%s PMU driver.\n", x86_pmu.name);
@@ -2097,14 +2095,17 @@ static int __init init_hw_perf_events(void) 
		quirk->func();



	if (!x86_pmu.intel_ctrl)

		x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;

		x86_pmu.intel_ctrl = x86_pmu.cntr_mask64;



	if (!x86_pmu.config_mask)

		x86_pmu.config_mask = X86_RAW_EVENT_MASK;



	perf_events_lapic_init();

	register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");



	unconstrained = (struct event_constraint)

		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,

				   0, x86_pmu.num_counters, 0, 0);

		__EVENT_CONSTRAINT(0, x86_pmu.cntr_mask64,

				   0, x86_pmu_num_counters(NULL), 0, 0);



	x86_pmu_format_group.attrs = x86_pmu.format_attrs;
@@ -2113,11 +2114,8 @@ static int __init init_hw_perf_events(void) 


	pmu.attr_update = x86_pmu.attr_update;



	if (!is_hybrid()) {

		x86_pmu_show_pmu_cap(x86_pmu.num_counters,

				     x86_pmu.num_counters_fixed,

				     x86_pmu.intel_ctrl);

	}

	if (!is_hybrid())

		x86_pmu_show_pmu_cap(NULL);



	if (!x86_pmu.read)

		x86_pmu.read = _x86_pmu_read;
@@ -2481,10 +2479,10 @@ void perf_clear_dirty_counters(void) 
	for_each_set_bit(i, cpuc->dirty, X86_PMC_IDX_MAX) {

		if (i >= INTEL_PMC_IDX_FIXED) {

			/* Metrics and fake events don't have corresponding HW counters. */

			if ((i - INTEL_PMC_IDX_FIXED) >= hybrid(cpuc->pmu, num_counters_fixed))

			if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask)))

				continue;



			wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + (i - INTEL_PMC_IDX_FIXED), 0);

			wrmsrl(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0);

		} else {

			wrmsrl(x86_pmu_event_addr(i), 0);

		}
@@ -2986,8 +2984,8 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 
	 * base PMU holds the correct number of counters for P-cores.

	 */

	cap->version		= x86_pmu.version;

	cap->num_counters_gp	= x86_pmu.num_counters;

	cap->num_counters_fixed	= x86_pmu.num_counters_fixed;

	cap->num_counters_gp	= x86_pmu_num_counters(NULL);

	cap->num_counters_fixed	= x86_pmu_num_counters_fixed(NULL);

	cap->bit_width_gp	= x86_pmu.cntval_bits;

	cap->bit_width_fixed	= x86_pmu.cntval_bits;

	cap->events_mask	= (unsigned int)x86_pmu.events_maskl;

arch/x86/events/intel/core.c

+327 −99

File changed.

Preview size limit exceeded, changes collapsed.

arch/x86/events/intel/cstate.c

+26 −13
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ 
 *	MSR_CORE_C1_RES: CORE C1 Residency Counter

 *			 perf code: 0x00

 *			 Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL

 *					  MTL,SRF,GRR

 *					  MTL,SRF,GRR,ARL,LNL

 *			 Scope: Core (each processor core has a MSR)

 *	MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter

 *			       perf code: 0x01
@@ -53,50 +53,50 @@ 
 *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,

 *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,

 *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,

 *						GRR

 *						GRR,ARL,LNL

 *			       Scope: Core

 *	MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter

 *			       perf code: 0x03

 *			       Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,

 *						ICL,TGL,RKL,ADL,RPL,MTL

 *						ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL

 *			       Scope: Core

 *	MSR_PKG_C2_RESIDENCY:  Package C2 Residency Counter.

 *			       perf code: 0x00

 *			       Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,

 *						KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,

 *						RPL,SPR,MTL

 *						RPL,SPR,MTL,ARL,LNL

 *			       Scope: Package (physical package)

 *	MSR_PKG_C3_RESIDENCY:  Package C3 Residency Counter.

 *			       perf code: 0x01

 *			       Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,

 *						GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL,

 *						ADL,RPL,MTL

 *						ADL,RPL,MTL,ARL,LNL

 *			       Scope: Package (physical package)

 *	MSR_PKG_C6_RESIDENCY:  Package C6 Residency Counter.

 *			       perf code: 0x02

 *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,

 *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,

 *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF

 *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,

 *						ARL,LNL

 *			       Scope: Package (physical package)

 *	MSR_PKG_C7_RESIDENCY:  Package C7 Residency Counter.

 *			       perf code: 0x03

 *			       Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL,

 *						KBL,CML,ICL,TGL,RKL,ADL,RPL,MTL

 *						KBL,CML,ICL,TGL,RKL

 *			       Scope: Package (physical package)

 *	MSR_PKG_C8_RESIDENCY:  Package C8 Residency Counter.

 *			       perf code: 0x04

 *			       Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,

 *						ADL,RPL,MTL

 *						ADL,RPL,MTL,ARL

 *			       Scope: Package (physical package)

 *	MSR_PKG_C9_RESIDENCY:  Package C9 Residency Counter.

 *			       perf code: 0x05

 *			       Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,

 *						ADL,RPL,MTL

 *			       Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL

 *			       Scope: Package (physical package)

 *	MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.

 *			       perf code: 0x06

 *			       Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,

 *						TNT,RKL,ADL,RPL,MTL

 *						TNT,RKL,ADL,RPL,MTL,ARL,LNL

 *			       Scope: Package (physical package)

 *	MSR_MODULE_C6_RES_MS:  Module C6 Residency Counter.

 *			       perf code: 0x00
@@ -637,9 +637,18 @@ static const struct cstate_model adl_cstates __initconst = { 
	.pkg_events		= BIT(PERF_CSTATE_PKG_C2_RES) |

				  BIT(PERF_CSTATE_PKG_C3_RES) |

				  BIT(PERF_CSTATE_PKG_C6_RES) |

				  BIT(PERF_CSTATE_PKG_C7_RES) |

				  BIT(PERF_CSTATE_PKG_C8_RES) |

				  BIT(PERF_CSTATE_PKG_C9_RES) |

				  BIT(PERF_CSTATE_PKG_C10_RES),

};



static const struct cstate_model lnl_cstates __initconst = {

	.core_events		= BIT(PERF_CSTATE_CORE_C1_RES) |

				  BIT(PERF_CSTATE_CORE_C6_RES) |

				  BIT(PERF_CSTATE_CORE_C7_RES),



	.pkg_events		= BIT(PERF_CSTATE_PKG_C2_RES) |

				  BIT(PERF_CSTATE_PKG_C3_RES) |

				  BIT(PERF_CSTATE_PKG_C6_RES) |

				  BIT(PERF_CSTATE_PKG_C10_RES),

};
@@ -763,6 +772,10 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { 
	X86_MATCH_VFM(INTEL_RAPTORLAKE_S,	&adl_cstates),

	X86_MATCH_VFM(INTEL_METEORLAKE,		&adl_cstates),

	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&adl_cstates),

	X86_MATCH_VFM(INTEL_ARROWLAKE,		&adl_cstates),

	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&adl_cstates),

	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&adl_cstates),

	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_cstates),

	{ },

};

MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);