Commit 87a194e6 authored by Bjorn Helgaas's avatar Bjorn Helgaas
Browse files

Merge branch 'pci/endpoint' 

- Convert the endpoint doorbell test to use a threaded IRQ to fix a
  'sleeping while atomic' issue (Bhanu Seshu Kumar Valluri)

- Add endpoint VNTB MSI doorbell support to reduce latency between host and
  endpoint (Frank Li)

* pci/endpoint:
  PCI: endpoint: pci-epf-vntb: Add MSI doorbell support
  PCI: endpoint: Add pci_epf_assign_bar_space() API
  PCI: endpoint: Add pci_epf_get_required_bar_size() helper
  PCI: endpoint: Rename 'epf_bar::aligned_size' to 'epf_bar:mem_size'
  PCI: endpoint: pci-epf-test: Fix sleeping function being called from atomic context
parents f26a75c9 dc693d60

drivers/pci/endpoint/functions/pci-epf-test.c

+3 −2
Original line number Diff line number Diff line
@@ -730,7 +730,8 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test, 
	if (bar < BAR_0)

		goto err_doorbell_cleanup;



	ret = request_irq(epf->db_msg[0].virq, pci_epf_test_doorbell_handler, 0,

	ret = request_threaded_irq(epf->db_msg[0].virq, NULL,

				   pci_epf_test_doorbell_handler, IRQF_ONESHOT,

				   "pci-ep-test-doorbell", epf_test);

	if (ret) {

		dev_err(&epf->dev,

drivers/pci/endpoint/functions/pci-epf-vntb.c

+136 −17
Original line number Diff line number Diff line
@@ -36,11 +36,13 @@ 
 * PCIe Root Port                        PCI EP

 */



#include <linux/atomic.h>

#include <linux/delay.h>

#include <linux/io.h>

#include <linux/module.h>

#include <linux/slab.h>



#include <linux/pci-ep-msi.h>

#include <linux/pci-epc.h>

#include <linux/pci-epf.h>

#include <linux/ntb.h>
@@ -126,12 +128,13 @@ struct epf_ntb { 
	u32 db_count;

	u32 spad_count;

	u64 mws_size[MAX_MW];

	u64 db;

	atomic64_t db;

	u32 vbus_number;

	u16 vntb_pid;

	u16 vntb_vid;



	bool linkup;

	bool msi_doorbell;

	u32 spad_size;



	enum pci_barno epf_ntb_bar[VNTB_BAR_NUM];
@@ -258,9 +261,9 @@ static void epf_ntb_cmd_handler(struct work_struct *work) 


	ntb = container_of(work, struct epf_ntb, cmd_handler.work);



	for (i = 1; i < ntb->db_count; i++) {

	for (i = 1; i < ntb->db_count && !ntb->msi_doorbell; i++) {

		if (ntb->epf_db[i]) {

			ntb->db |= 1 << (i - 1);

			atomic64_or(1 << (i - 1), &ntb->db);

			ntb_db_event(&ntb->ntb, i);

			ntb->epf_db[i] = 0;

		}
@@ -319,7 +322,21 @@ static void epf_ntb_cmd_handler(struct work_struct *work) 


reset_handler:

	queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,

			   msecs_to_jiffies(5));

			   ntb->msi_doorbell ? msecs_to_jiffies(500) : msecs_to_jiffies(5));

}



static irqreturn_t epf_ntb_doorbell_handler(int irq, void *data)

{

	struct epf_ntb *ntb = data;

	int i;



	for (i = 1; i < ntb->db_count; i++)

		if (irq == ntb->epf->db_msg[i].virq) {

			atomic64_or(1 << (i - 1), &ntb->db);

			ntb_db_event(&ntb->ntb, i);

		}



	return IRQ_HANDLED;

}



/**
@@ -500,6 +517,94 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb) 
	return 0;

}



static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,

					    struct pci_epf_bar *db_bar,

					    const struct pci_epc_features *epc_features,

					    enum pci_barno barno)

{

	struct pci_epf *epf = ntb->epf;

	dma_addr_t low, high;

	struct msi_msg *msg;

	size_t sz;

	int ret;

	int i;



	ret = pci_epf_alloc_doorbell(epf,  ntb->db_count);

	if (ret)

		return ret;



	for (i = 0; i < ntb->db_count; i++) {

		ret = request_irq(epf->db_msg[i].virq, epf_ntb_doorbell_handler,

				  0, "pci_epf_vntb_db", ntb);



		if (ret) {

			dev_err(&epf->dev,

				"Failed to request doorbell IRQ: %d\n",

				epf->db_msg[i].virq);

			goto err_free_irq;

		}

	}



	msg = &epf->db_msg[0].msg;



	high = 0;

	low = (u64)msg->address_hi << 32 | msg->address_lo;



	for (i = 0; i < ntb->db_count; i++) {

		struct msi_msg *msg = &epf->db_msg[i].msg;

		dma_addr_t addr = (u64)msg->address_hi << 32 | msg->address_lo;



		low = min(low, addr);

		high = max(high, addr);

	}



	sz = high - low + sizeof(u32);



	ret = pci_epf_assign_bar_space(epf, sz, barno, epc_features, 0, low);

	if (ret) {

		dev_err(&epf->dev, "Failed to assign Doorbell BAR space\n");

		goto err_free_irq;

	}



	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,

			      ntb->epf->vfunc_no, db_bar);

	if (ret) {

		dev_err(&epf->dev, "Failed to set Doorbell BAR\n");

		goto err_free_irq;

	}



	for (i = 0; i < ntb->db_count; i++) {

		struct msi_msg *msg = &epf->db_msg[i].msg;

		dma_addr_t addr;

		size_t offset;



		ret = pci_epf_align_inbound_addr(epf, db_bar->barno,

				((u64)msg->address_hi << 32) | msg->address_lo,

				&addr, &offset);



		if (ret) {

			ntb->msi_doorbell = false;

			goto err_free_irq;

		}



		ntb->reg->db_data[i] = msg->data;

		ntb->reg->db_offset[i] = offset;

	}



	ntb->reg->db_entry_size = 0;



	ntb->msi_doorbell = true;



	return 0;



err_free_irq:

	for (i--; i >= 0; i--)

		free_irq(epf->db_msg[i].virq, ntb);



	pci_epf_free_doorbell(ntb->epf);

	return ret;

}



/**

 * epf_ntb_db_bar_init() - Configure Doorbell window BARs

 * @ntb: NTB device that facilitates communication between HOST and VHOST
@@ -520,7 +625,11 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb) 
					    ntb->epf->func_no,

					    ntb->epf->vfunc_no);

	barno = ntb->epf_ntb_bar[BAR_DB];

	epf_bar = &ntb->epf->bar[barno];



	ret = epf_ntb_db_bar_init_msi_doorbell(ntb, epf_bar, epc_features, barno);

	if (ret) {

		/* fall back to polling mode */

		mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0);

		if (!mw_addr) {

			dev_err(dev, "Failed to allocate OB address\n");
@@ -529,13 +638,13 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb) 


		ntb->epf_db = mw_addr;



	epf_bar = &ntb->epf->bar[barno];



	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);

		ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,

				      ntb->epf->vfunc_no, epf_bar);

		if (ret) {

			dev_err(dev, "Doorbell BAR set failed\n");

			goto err_alloc_peer_mem;

		}

	}

	return ret;



err_alloc_peer_mem:
@@ -554,6 +663,16 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb) 
{

	enum pci_barno barno;



	if (ntb->msi_doorbell) {

		int i;



		for (i = 0; i < ntb->db_count; i++)

			free_irq(ntb->epf->db_msg[i].virq, ntb);

	}



	if (ntb->epf->db_msg)

		pci_epf_free_doorbell(ntb->epf);



	barno = ntb->epf_ntb_bar[BAR_DB];

	pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);

	pci_epc_clear_bar(ntb->epf->epc,
@@ -1268,7 +1387,7 @@ static u64 vntb_epf_db_read(struct ntb_dev *ndev) 
{

	struct epf_ntb *ntb = ntb_ndev(ndev);



	return ntb->db;

	return atomic64_read(&ntb->db);

}



static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
@@ -1308,7 +1427,7 @@ static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits) 
{

	struct epf_ntb *ntb = ntb_ndev(ndev);



	ntb->db &= ~db_bits;

	atomic64_and(~db_bits, &ntb->db);

	return 0;

}

drivers/pci/endpoint/pci-epf-core.c

+127 −32
Original line number Diff line number Diff line
@@ -208,6 +208,48 @@ void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) 
}

EXPORT_SYMBOL_GPL(pci_epf_remove_vepf);



static int pci_epf_get_required_bar_size(struct pci_epf *epf, size_t *bar_size,

				size_t *aligned_mem_size,

				enum pci_barno bar,

				const struct pci_epc_features *epc_features,

				enum pci_epc_interface_type type)

{

	u64 bar_fixed_size = epc_features->bar[bar].fixed_size;

	size_t align = epc_features->align;

	size_t size = *bar_size;



	if (size < 128)

		size = 128;



	/* According to PCIe base spec, min size for a resizable BAR is 1 MB. */

	if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M)

		size = SZ_1M;



	if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) {

		if (size > bar_fixed_size) {

			dev_err(&epf->dev,

				"requested BAR size is larger than fixed size\n");

			return -ENOMEM;

		}

		size = bar_fixed_size;

	} else {

		/* BAR size must be power of two */

		size = roundup_pow_of_two(size);

	}



	*bar_size = size;



	/*

	 * The EPC's BAR start address must meet alignment requirements. In most

	 * cases, the alignment will match the BAR size. However, differences

	 * can occur—for example, when the fixed BAR size (e.g., 128 bytes) is

	 * smaller than the required alignment (e.g., 4 KB).

	 */

	*aligned_mem_size = align ? ALIGN(size, align) : size;



	return 0;

}



/**

 * pci_epf_free_space() - free the allocated PCI EPF register space

 * @epf: the EPF device from whom to free the memory
@@ -236,13 +278,13 @@ void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, 
	}



	dev = epc->dev.parent;

	dma_free_coherent(dev, epf_bar[bar].aligned_size, addr,

	dma_free_coherent(dev, epf_bar[bar].mem_size, addr,

			  epf_bar[bar].phys_addr);



	epf_bar[bar].phys_addr = 0;

	epf_bar[bar].addr = NULL;

	epf_bar[bar].size = 0;

	epf_bar[bar].aligned_size = 0;

	epf_bar[bar].mem_size = 0;

	epf_bar[bar].barno = 0;

	epf_bar[bar].flags = 0;

}
@@ -264,40 +306,16 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, 
			  const struct pci_epc_features *epc_features,

			  enum pci_epc_interface_type type)

{

	u64 bar_fixed_size = epc_features->bar[bar].fixed_size;

	size_t aligned_size, align = epc_features->align;

	struct pci_epf_bar *epf_bar;

	dma_addr_t phys_addr;

	struct pci_epc *epc;

	struct device *dev;

	size_t mem_size;

	void *space;



	if (size < 128)

		size = 128;



	/* According to PCIe base spec, min size for a resizable BAR is 1 MB. */

	if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M)

		size = SZ_1M;



	if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) {

		if (size > bar_fixed_size) {

			dev_err(&epf->dev,

				"requested BAR size is larger than fixed size\n");

	if (pci_epf_get_required_bar_size(epf, &size, &mem_size, bar,

					  epc_features, type))

		return NULL;

		}

		size = bar_fixed_size;

	} else {

		/* BAR size must be power of two */

		size = roundup_pow_of_two(size);

	}



	/*

	 * Allocate enough memory to accommodate the iATU alignment

	 * requirement.  In most cases, this will be the same as .size but

	 * it might be different if, for example, the fixed size of a BAR

	 * is smaller than align.

	 */

	aligned_size = align ? ALIGN(size, align) : size;



	if (type == PRIMARY_INTERFACE) {

		epc = epf->epc;
@@ -308,7 +326,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, 
	}



	dev = epc->dev.parent;

	space = dma_alloc_coherent(dev, aligned_size, &phys_addr, GFP_KERNEL);

	space = dma_alloc_coherent(dev, mem_size, &phys_addr, GFP_KERNEL);

	if (!space) {

		dev_err(dev, "failed to allocate mem space\n");

		return NULL;
@@ -317,7 +335,7 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, 
	epf_bar[bar].phys_addr = phys_addr;

	epf_bar[bar].addr = space;

	epf_bar[bar].size = size;

	epf_bar[bar].aligned_size = aligned_size;

	epf_bar[bar].mem_size = mem_size;

	epf_bar[bar].barno = bar;

	if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)

		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
@@ -328,6 +346,83 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, 
}

EXPORT_SYMBOL_GPL(pci_epf_alloc_space);



/**

 * pci_epf_assign_bar_space() - Assign PCI EPF BAR space

 * @epf: EPF device to assign the BAR memory

 * @size: Size of the memory that has to be assigned

 * @bar: BAR number for which the memory is assigned

 * @epc_features: Features provided by the EPC specific to this EPF

 * @type: Identifies if the assignment is for primary EPC or secondary EPC

 * @bar_addr: Address to be assigned for the @bar

 *

 * Invoke to assign memory for the PCI EPF BAR.

 * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR

 * can only be a 64-bit BAR, or if the requested size is larger than 2 GB.

 */

int pci_epf_assign_bar_space(struct pci_epf *epf, size_t size,

			     enum pci_barno bar,

			     const struct pci_epc_features *epc_features,

			     enum pci_epc_interface_type type,

			     dma_addr_t bar_addr)

{

	size_t bar_size, aligned_mem_size;

	struct pci_epf_bar *epf_bar;

	dma_addr_t limit;

	int pos;



	if (!size)

		return -EINVAL;



	limit = bar_addr + size - 1;



	/*

	 *  Bits:		15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0

	 *  bar_addr:		U  U  U  U  U  U  0 X X X X X X X X X

	 *  limit:		U  U  U  U  U  U  1 X X X X X X X X X

	 *

	 *  bar_addr^limit	0  0  0  0  0  0  1 X X X X X X X X X

	 *

	 *  U: unchanged address bits in range [bar_addr, limit]

	 *  X: bit 0 or 1

	 *

	 *  (bar_addr^limit) & BIT_ULL(pos) will find the first set bit from MSB

	 *  (pos). And value of (2 ^ pos) should be able to cover the BAR range.

	 */

	for (pos = 8 * sizeof(dma_addr_t) - 1; pos > 0; pos--)

		if ((limit ^ bar_addr) & BIT_ULL(pos))

			break;



	if (pos == 8 * sizeof(dma_addr_t) - 1)

		return -EINVAL;



	bar_size = BIT_ULL(pos + 1);

	if (pci_epf_get_required_bar_size(epf, &bar_size, &aligned_mem_size,

					  bar, epc_features, type))

		return -ENOMEM;



	if (type == PRIMARY_INTERFACE)

		epf_bar = epf->bar;

	else

		epf_bar = epf->sec_epc_bar;



	epf_bar[bar].phys_addr = ALIGN_DOWN(bar_addr, aligned_mem_size);



	if (epf_bar[bar].phys_addr + bar_size < limit)

		return -ENOMEM;



	epf_bar[bar].addr = NULL;

	epf_bar[bar].size = bar_size;

	epf_bar[bar].mem_size = aligned_mem_size;

	epf_bar[bar].barno = bar;

	if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)

		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;

	else

		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32;



	return 0;

}

EXPORT_SYMBOL_GPL(pci_epf_assign_bar_space);



static void pci_epf_remove_cfs(struct pci_epf_driver *driver)

{

	struct config_group *group, *tmp;

include/linux/pci-epf.h

+9 −3
Original line number Diff line number Diff line
@@ -115,7 +115,7 @@ struct pci_epf_driver { 
 * @phys_addr: physical address that should be mapped to the BAR

 * @addr: virtual address corresponding to the @phys_addr

 * @size: the size of the address space present in BAR

 * @aligned_size: the size actually allocated to accommodate the iATU alignment

 * @mem_size: the size actually allocated to accommodate the iATU alignment

 *            requirement

 * @barno: BAR number

 * @flags: flags that are set for the BAR
@@ -124,7 +124,7 @@ struct pci_epf_bar { 
	dma_addr_t	phys_addr;

	void		*addr;

	size_t		size;

	size_t		aligned_size;

	size_t		mem_size;

	enum pci_barno	barno;

	int		flags;

};
@@ -242,6 +242,12 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, 
void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar,

			enum pci_epc_interface_type type);



int pci_epf_assign_bar_space(struct pci_epf *epf, size_t size,

			     enum pci_barno bar,

			     const struct pci_epc_features *epc_features,

			     enum pci_epc_interface_type type,

			     dma_addr_t bar_addr);



int pci_epf_align_inbound_addr(struct pci_epf *epf, enum pci_barno bar,

			       u64 addr, dma_addr_t *base, size_t *off);

int pci_epf_bind(struct pci_epf *epf);