Commit 4d07a053 authored by Dave Jiang's avatar Dave Jiang Committed by Dan Williams
Browse files

cxl: Calculate and store PCI link latency for the downstream ports 



The latency is calculated by dividing the flit size over the bandwidth. Add
support to retrieve the flit size for the CXL switch device and calculate
the latency of the PCIe link. Cache the latency number with cxl_dport.

Reviewed-by: default avatarJonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: default avatarDave Jiang <dave.jiang@intel.com>
Link: https://lore.kernel.org/r/170319621931.2212653.6800240203604822886.stgit@djiang5-mobl3


Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
parent 79081590

drivers/cxl/core/core.h

+2 −0
Original line number Diff line number Diff line
@@ -88,4 +88,6 @@ enum cxl_poison_trace_type { 
	CXL_POISON_TRACE_CLEAR,

};



long cxl_pci_get_latency(struct pci_dev *pdev);



#endif /* __CXL_CORE_H__ */

drivers/cxl/core/pci.c

+36 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/* Copyright(c) 2021 Intel Corporation. All rights reserved. */

#include <linux/units.h>

#include <linux/io-64-nonatomic-lo-hi.h>

#include <linux/device.h>

#include <linux/delay.h>
@@ -979,3 +980,38 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, 
	return PCI_ERS_RESULT_NEED_RESET;

}

EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL);



static int cxl_flit_size(struct pci_dev *pdev)

{

	if (cxl_pci_flit_256(pdev))

		return 256;



	return 68;

}



/**

 * cxl_pci_get_latency - calculate the link latency for the PCIe link

 * @pdev: PCI device

 *

 * return: calculated latency or 0 for no latency

 *

 * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation

 * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency

 * LinkProgationLatency is negligible, so 0 will be used

 * RetimerLatency is assumed to be negligible and 0 will be used

 * FlitLatency = FlitSize / LinkBandwidth

 * FlitSize is defined by spec. CXL rev3.0 4.2.1.

 * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used.

 * The FlitLatency is converted to picoseconds.

 */

long cxl_pci_get_latency(struct pci_dev *pdev)

{

	long bw;



	bw = pcie_link_speed_mbps(pdev);

	if (bw < 0)

		return 0;

	bw /= BITS_PER_BYTE;



	return cxl_flit_size(pdev) * MEGA / bw;

}

drivers/cxl/core/port.c

+6 −0
Original line number Diff line number Diff line
@@ -854,6 +854,9 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, 
	if (rc)

		return ERR_PTR(rc);



	if (parent_dport && dev_is_pci(uport_dev))

		port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));



	return port;



err:
@@ -1137,6 +1140,9 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, 
	if (rc)

		return ERR_PTR(rc);



	if (dev_is_pci(dport_dev))

		dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));



	return dport;

}

drivers/cxl/cxl.h

+4 −0
Original line number Diff line number Diff line
@@ -591,6 +591,7 @@ struct cxl_dax_region { 
 * @depth: How deep this port is relative to the root. depth 0 is the root.

 * @cdat: Cached CDAT data

 * @cdat_available: Should a CDAT attribute be available in sysfs

 * @pci_latency: Upstream latency in picoseconds

 */

struct cxl_port {

	struct device dev;
@@ -613,6 +614,7 @@ struct cxl_port { 
		size_t length;

	} cdat;

	bool cdat_available;

	long pci_latency;

};



struct cxl_root_ops {
@@ -659,6 +661,7 @@ struct cxl_rcrb_info { 
 * @port: reference to cxl_port that contains this downstream port

 * @regs: Dport parsed register blocks

 * @sw_coord: access coordinates (performance) for switch from CDAT

 * @link_latency: calculated PCIe downstream latency

 */

struct cxl_dport {

	struct device *dport_dev;
@@ -669,6 +672,7 @@ struct cxl_dport { 
	struct cxl_port *port;

	struct cxl_regs regs;

	struct access_coordinate sw_coord;

	long link_latency;

};



/**

drivers/cxl/cxlpci.h

+13 −0
Original line number Diff line number Diff line
@@ -85,6 +85,19 @@ struct cdat_entry_header { 
	__le16 length;

} __packed;



/*

 * CXL v3.0 6.2.3 Table 6-4

 * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits

 * mode, otherwise it's 68B flits mode.

 */

static inline bool cxl_pci_flit_256(struct pci_dev *pdev)

{

	u16 lnksta2;



	pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2);

	return lnksta2 & PCI_EXP_LNKSTA2_FLIT;

}



int devm_cxl_port_enumerate_dports(struct cxl_port *port);

struct cxl_dev_state;

int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,