Commit 71396cfa authored by Ilkka Koskinen's avatar Ilkka Koskinen Committed by Will Deacon
Browse files

perf/dwc_pcie: Support counting multiple lane events in parallel 



While Designware PCIe PMU allows to count only one time based event
at a time, it allows to count all the lane events simultaneously.
After the patch one is able to count a group of lane events:

  $  perf stat -e '{dwc_rootport/tx_memory_write,lane=1/,dwc_rootport/rx_memory_read,lane=0/}' dd if=/dev/nvme0n1 of=/dev/null bs=1M count=1

Earlier the events wouldn't have been counted successfully.

Signed-off-by: default avatarIlkka Koskinen <ilkka@os.amperecomputing.com>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 1e558fb3

Documentation/admin-guide/perf/dwc_pcie_pmu.rst

+2 −2
Original line number Diff line number Diff line
@@ -16,8 +16,8 @@ provides the following two features: 


- one 64-bit counter for Time Based Analysis (RX/TX data throughput and

  time spent in each low-power LTSSM state) and

- one 32-bit counter for Event Counting (error and non-error events for

  a specified lane)

- one 32-bit counter per event for Event Counting (error and non-error

  events for a specified lane)



Note: There is no interrupt for counter overflow.

drivers/perf/dwc_pcie_pmu.c

+130 −31
Original line number Diff line number Diff line
@@ -39,6 +39,10 @@ 
#define DWC_PCIE_EVENT_CLEAR			GENMASK(1, 0)

#define DWC_PCIE_EVENT_PER_CLEAR		0x1



/* Event Selection Field has two subfields */

#define DWC_PCIE_CNT_EVENT_SEL_GROUP		GENMASK(11, 8)

#define DWC_PCIE_CNT_EVENT_SEL_EVID		GENMASK(7, 0)



#define DWC_PCIE_EVENT_CNT_DATA			0xC



#define DWC_PCIE_TIME_BASED_ANAL_CTL		0x10
@@ -73,6 +77,10 @@ enum dwc_pcie_event_type { 
	DWC_PCIE_EVENT_TYPE_MAX,

};



#define DWC_PCIE_LANE_GROUP_6 6

#define DWC_PCIE_LANE_GROUP_7 7

#define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256



#define DWC_PCIE_LANE_EVENT_MAX_PERIOD		GENMASK_ULL(31, 0)

#define DWC_PCIE_MAX_PERIOD			GENMASK_ULL(63, 0)
@@ -82,8 +90,11 @@ struct dwc_pcie_pmu { 
	u16			ras_des_offset;

	u32			nr_lanes;



	/* Groups #6 and #7 */

	DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);

	struct perf_event	*time_based_event;



	struct hlist_node	cpuhp_node;

	struct perf_event	*event[DWC_PCIE_EVENT_TYPE_MAX];

	int			on_cpu;

};
@@ -246,19 +257,26 @@ static const struct attribute_group *dwc_pcie_attr_groups[] = { 
};



static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,

					   struct perf_event *event,

					   bool enable)

{

	struct pci_dev *pdev = pcie_pmu->pdev;

	u16 ras_des_offset = pcie_pmu->ras_des_offset;

	int event_id = DWC_PCIE_EVENT_ID(event);

	int lane = DWC_PCIE_EVENT_LANE(event);

	u32 ctrl;



	ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |

		FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |

		FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);



	if (enable)

		pci_clear_and_set_config_dword(pdev,

					ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,

					DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);

		ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);

	else

		pci_clear_and_set_config_dword(pdev,

					ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,

					DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);

		ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);



	pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,

			       ctrl);

}



static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
@@ -276,11 +294,22 @@ static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) 
{

	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);

	struct pci_dev *pdev = pcie_pmu->pdev;

	int event_id = DWC_PCIE_EVENT_ID(event);

	int lane = DWC_PCIE_EVENT_LANE(event);

	u16 ras_des_offset = pcie_pmu->ras_des_offset;

	u32 val;

	u32 val, ctrl;



	ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |

		FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |

		FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);

	pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,

			       ctrl);

	pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);



	ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);

	pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,

			       ctrl);



	return val;

}
@@ -329,26 +358,77 @@ static void dwc_pcie_pmu_event_update(struct perf_event *event) 
{

	struct hw_perf_event *hwc = &event->hw;

	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);

	u64 delta, prev, now = 0;

	u64 delta, prev, now;



	if (type == DWC_PCIE_LANE_EVENT) {

		now = dwc_pcie_pmu_read_lane_event_counter(event) &

			DWC_PCIE_LANE_EVENT_MAX_PERIOD;

		local64_add(now, &event->count);

		return;

	}



	do {

		prev = local64_read(&hwc->prev_count);



		if (type == DWC_PCIE_LANE_EVENT)

			now = dwc_pcie_pmu_read_lane_event_counter(event);

		else if (type == DWC_PCIE_TIME_BASE_EVENT)

		now = dwc_pcie_pmu_read_time_based_counter(event);



	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);



	delta = (now - prev) & DWC_PCIE_MAX_PERIOD;

	/* 32-bit counter for Lane Event Counting */

	if (type == DWC_PCIE_LANE_EVENT)

		delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD;



	local64_add(delta, &event->count);

}



static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event,

						unsigned long val_lane_events[])

{

	int event_id, event_nr, group;



	event_id = DWC_PCIE_EVENT_ID(event);

	event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);

	group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id);



	if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7)

		return -EINVAL;



	group -= DWC_PCIE_LANE_GROUP_6;



	if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,

			     val_lane_events))

		return -EINVAL;



	return 0;

}



static int dwc_pcie_pmu_validate_group(struct perf_event *event)

{

	struct perf_event *sibling, *leader = event->group_leader;

	DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);

	bool time_event;

	int type;



	type = DWC_PCIE_EVENT_TYPE(leader);

	if (type == DWC_PCIE_TIME_BASE_EVENT)

		time_event = true;

	else

		if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events))

			return -ENOSPC;



	for_each_sibling_event(sibling, leader) {

		type = DWC_PCIE_EVENT_TYPE(sibling);

		if (type == DWC_PCIE_TIME_BASE_EVENT) {

			if (time_event)

				return -ENOSPC;



			time_event = true;

			continue;

		}



		if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events))

			return -ENOSPC;

	}



	return 0;

}



static int dwc_pcie_pmu_event_init(struct perf_event *event)

{

	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
@@ -367,10 +447,6 @@ static int dwc_pcie_pmu_event_init(struct perf_event *event) 
	if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)

		return -EINVAL;



	if (event->group_leader != event &&

	    !is_software_event(event->group_leader))

		return -EINVAL;



	for_each_sibling_event(sibling, event->group_leader) {

		if (sibling->pmu != event->pmu && !is_software_event(sibling))

			return -EINVAL;
@@ -385,6 +461,9 @@ static int dwc_pcie_pmu_event_init(struct perf_event *event) 
			return -EINVAL;

	}



	if (dwc_pcie_pmu_validate_group(event))

		return -ENOSPC;



	event->cpu = pcie_pmu->on_cpu;



	return 0;
@@ -400,7 +479,7 @@ static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) 
	local64_set(&hwc->prev_count, 0);



	if (type == DWC_PCIE_LANE_EVENT)

		dwc_pcie_pmu_lane_event_enable(pcie_pmu, true);

		dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true);

	else if (type == DWC_PCIE_TIME_BASE_EVENT)

		dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);

}
@@ -414,12 +493,13 @@ static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) 
	if (event->hw.state & PERF_HES_STOPPED)

		return;



	dwc_pcie_pmu_event_update(event);



	if (type == DWC_PCIE_LANE_EVENT)

		dwc_pcie_pmu_lane_event_enable(pcie_pmu, false);

		dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false);

	else if (type == DWC_PCIE_TIME_BASE_EVENT)

		dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);



	dwc_pcie_pmu_event_update(event);

	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;

}
@@ -434,14 +514,17 @@ static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 
	u16 ras_des_offset = pcie_pmu->ras_des_offset;

	u32 ctrl;



	/* one counter for each type and it is in use */

	if (pcie_pmu->event[type])

		return -ENOSPC;



	pcie_pmu->event[type] = event;

	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;



	if (type == DWC_PCIE_LANE_EVENT) {

		int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);

		int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -

			DWC_PCIE_LANE_GROUP_6;



		if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,

				     pcie_pmu->lane_events))

			return -ENOSPC;



		/* EVENT_COUNTER_DATA_REG needs clear manually */

		ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |

			FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
@@ -450,6 +533,11 @@ static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) 
		pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,

				       ctrl);

	} else if (type == DWC_PCIE_TIME_BASE_EVENT) {

		if (pcie_pmu->time_based_event)

			return -ENOSPC;



		pcie_pmu->time_based_event = event;



		/*

		 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely

		 * use it with any manually controlled duration. And it is
@@ -478,7 +566,18 @@ static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) 


	dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);

	perf_event_update_userpage(event);

	pcie_pmu->event[type] = NULL;



	if (type == DWC_PCIE_TIME_BASE_EVENT) {

		pcie_pmu->time_based_event = NULL;

	} else {

		int event_id = DWC_PCIE_EVENT_ID(event);

		int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);

		int group    = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -

			DWC_PCIE_LANE_GROUP_6;



		clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,

			  pcie_pmu->lane_events);

	}

}



static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)