Commit 4f1be396 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'dmaengine-fix-5.14' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine 

Pull dmaengine fixes from Vinod Koul:
 "A bunch of driver fixes, notably:

   - idxd driver fixes for submission race, driver remove sequence,
     setup sequence for MSIXPERM, array index and updating descriptor
     vector

   - usb-dmac, pm reference leak fix

   - xilinx_dma, read-after-free fix

   - uniphier-xdmac fix for using atomic readl_poll_timeout_atomic()

   - of-dma, router_xlate to return

   - imx-dma, generic dma fix"

* tag 'dmaengine-fix-5.14' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine:
  dmaengine: imx-dma: configure the generic DMA type to make it work
  dmaengine: of-dma: router_xlate to return -EPROBE_DEFER if controller is not yet available
  dmaengine: stm32-dmamux: Fix PM usage counter unbalance in stm32 dmamux ops
  dmaengine: stm32-dma: Fix PM usage counter imbalance in stm32 dma ops
  dmaengine: uniphier-xdmac: Use readl_poll_timeout_atomic() in atomic state
  dmaengine: idxd: fix submission race window
  dmaengine: idxd: fix sequence for pci driver remove() and shutdown()
  dmaengine: idxd: fix desc->vector that isn't being updated
  dmaengine: idxd: fix setup sequence for MSIXPERM table
  dmaengine: idxd: fix array index when int_handles are being used
  dmaengine: usb-dmac: Fix PM reference leak in usb_dmac_probe()
  dmaengine: xilinx_dma: Fix read-after-free bug when terminating transfers
parents b4b927fc 7199dded

drivers/dma/idxd/idxd.h

+14 −0
Original line number Diff line number Diff line
@@ -294,6 +294,14 @@ struct idxd_desc { 
	struct idxd_wq *wq;

};



/*

 * This is software defined error for the completion status. We overload the error code

 * that will never appear in completion status and only SWERR register.

 */

enum idxd_completion_status {

	IDXD_COMP_DESC_ABORT = 0xff,

};



#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)

#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
@@ -482,4 +490,10 @@ static inline void perfmon_init(void) {} 
static inline void perfmon_exit(void) {}

#endif



static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason)

{

	idxd_dma_complete_txd(desc, reason);

	idxd_free_desc(desc->wq, desc);

}



#endif

drivers/dma/idxd/init.c

+20 −10
Original line number Diff line number Diff line
@@ -102,6 +102,8 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) 
		spin_lock_init(&idxd->irq_entries[i].list_lock);

	}



	idxd_msix_perm_setup(idxd);



	irq_entry = &idxd->irq_entries[0];

	rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread,

				  0, "idxd-misc", irq_entry);
@@ -148,7 +150,6 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) 
	}



	idxd_unmask_error_interrupts(idxd);

	idxd_msix_perm_setup(idxd);

	return 0;



 err_wq_irqs:
@@ -162,6 +163,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) 
 err_misc_irq:

	/* Disable error interrupt generation */

	idxd_mask_error_interrupts(idxd);

	idxd_msix_perm_clear(idxd);

 err_irq_entries:

	pci_free_irq_vectors(pdev);

	dev_err(dev, "No usable interrupts\n");
@@ -758,32 +760,40 @@ static void idxd_shutdown(struct pci_dev *pdev) 
	for (i = 0; i < msixcnt; i++) {

		irq_entry = &idxd->irq_entries[i];

		synchronize_irq(irq_entry->vector);

		free_irq(irq_entry->vector, irq_entry);

		if (i == 0)

			continue;

		idxd_flush_pending_llist(irq_entry);

		idxd_flush_work_list(irq_entry);

	}



	idxd_msix_perm_clear(idxd);

	idxd_release_int_handles(idxd);

	pci_free_irq_vectors(pdev);

	pci_iounmap(pdev, idxd->reg_base);

	pci_disable_device(pdev);

	destroy_workqueue(idxd->wq);

	flush_workqueue(idxd->wq);

}



static void idxd_remove(struct pci_dev *pdev)

{

	struct idxd_device *idxd = pci_get_drvdata(pdev);

	struct idxd_irq_entry *irq_entry;

	int msixcnt = pci_msix_vec_count(pdev);

	int i;



	dev_dbg(&pdev->dev, "%s called\n", __func__);

	idxd_shutdown(pdev);

	if (device_pasid_enabled(idxd))

		idxd_disable_system_pasid(idxd);

	idxd_unregister_devices(idxd);

	perfmon_pmu_remove(idxd);



	for (i = 0; i < msixcnt; i++) {

		irq_entry = &idxd->irq_entries[i];

		free_irq(irq_entry->vector, irq_entry);

	}

	idxd_msix_perm_clear(idxd);

	idxd_release_int_handles(idxd);

	pci_free_irq_vectors(pdev);

	pci_iounmap(pdev, idxd->reg_base);

	iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);

	pci_disable_device(pdev);

	destroy_workqueue(idxd->wq);

	perfmon_pmu_remove(idxd);

	device_unregister(&idxd->conf_dev);

}



static struct pci_driver idxd_pci_driver = {

drivers/dma/idxd/irq.c

+18 −9
Original line number Diff line number Diff line
@@ -245,12 +245,6 @@ static inline bool match_fault(struct idxd_desc *desc, u64 fault_addr) 
	return false;

}



static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason)

{

	idxd_dma_complete_txd(desc, reason);

	idxd_free_desc(desc->wq, desc);

}



static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,

				     enum irq_work_type wtype,

				     int *processed, u64 data)
@@ -272,8 +266,16 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, 
		reason = IDXD_COMPLETE_DEV_FAIL;



	llist_for_each_entry_safe(desc, t, head, llnode) {

		if (desc->completion->status) {

			if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS)

		u8 status = desc->completion->status & DSA_COMP_STATUS_MASK;



		if (status) {

			if (unlikely(status == IDXD_COMP_DESC_ABORT)) {

				complete_desc(desc, IDXD_COMPLETE_ABORT);

				(*processed)++;

				continue;

			}



			if (unlikely(status != DSA_COMP_SUCCESS))

				match_fault(desc, data);

			complete_desc(desc, reason);

			(*processed)++;
@@ -329,7 +331,14 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry, 
	spin_unlock_irqrestore(&irq_entry->list_lock, flags);



	list_for_each_entry(desc, &flist, list) {

		if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS)

		u8 status = desc->completion->status & DSA_COMP_STATUS_MASK;



		if (unlikely(status == IDXD_COMP_DESC_ABORT)) {

			complete_desc(desc, IDXD_COMPLETE_ABORT);

			continue;

		}



		if (unlikely(status != DSA_COMP_SUCCESS))

			match_fault(desc, data);

		complete_desc(desc, reason);

	}

drivers/dma/idxd/submit.c

+70 −22
Original line number Diff line number Diff line
@@ -25,11 +25,10 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) 
	 * Descriptor completion vectors are 1...N for MSIX. We will round

	 * robin through the N vectors.

	 */

	wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;

	wq->vec_ptr = desc->vector = (wq->vec_ptr % idxd->num_wq_irqs) + 1;

	if (!idxd->int_handles) {

		desc->hw->int_handle = wq->vec_ptr;

	} else {

		desc->vector = wq->vec_ptr;

		/*

		 * int_handles are only for descriptor completion. However for device

		 * MSIX enumeration, vec 0 is used for misc interrupts. Therefore even
@@ -88,9 +87,64 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) 
	sbitmap_queue_clear(&wq->sbq, desc->id, cpu);

}



static struct idxd_desc *list_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,

					 struct idxd_desc *desc)

{

	struct idxd_desc *d, *n;



	lockdep_assert_held(&ie->list_lock);

	list_for_each_entry_safe(d, n, &ie->work_list, list) {

		if (d == desc) {

			list_del(&d->list);

			return d;

		}

	}



	/*

	 * At this point, the desc needs to be aborted is held by the completion

	 * handler where it has taken it off the pending list but has not added to the

	 * work list. It will be cleaned up by the interrupt handler when it sees the

	 * IDXD_COMP_DESC_ABORT for completion status.

	 */

	return NULL;

}



static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,

			     struct idxd_desc *desc)

{

	struct idxd_desc *d, *t, *found = NULL;

	struct llist_node *head;

	unsigned long flags;



	desc->completion->status = IDXD_COMP_DESC_ABORT;

	/*

	 * Grab the list lock so it will block the irq thread handler. This allows the

	 * abort code to locate the descriptor need to be aborted.

	 */

	spin_lock_irqsave(&ie->list_lock, flags);

	head = llist_del_all(&ie->pending_llist);

	if (head) {

		llist_for_each_entry_safe(d, t, head, llnode) {

			if (d == desc) {

				found = desc;

				continue;

			}

			list_add_tail(&desc->list, &ie->work_list);

		}

	}



	if (!found)

		found = list_abort_desc(wq, ie, desc);

	spin_unlock_irqrestore(&ie->list_lock, flags);



	if (found)

		complete_desc(found, IDXD_COMPLETE_ABORT);

}



int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)

{

	struct idxd_device *idxd = wq->idxd;

	struct idxd_irq_entry *ie = NULL;

	void __iomem *portal;

	int rc;
@@ -108,6 +162,16 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) 
	 * even on UP because the recipient is a device.

	 */

	wmb();



	/*

	 * Pending the descriptor to the lockless list for the irq_entry

	 * that we designated the descriptor to.

	 */

	if (desc->hw->flags & IDXD_OP_FLAG_RCI) {

		ie = &idxd->irq_entries[desc->vector];

		llist_add(&desc->llnode, &ie->pending_llist);

	}



	if (wq_dedicated(wq)) {

		iosubmit_cmds512(portal, desc->hw, 1);

	} else {
@@ -118,29 +182,13 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) 
		 * device is not accepting descriptor at all.

		 */

		rc = enqcmds(portal, desc->hw);

		if (rc < 0)

		if (rc < 0) {

			if (ie)

				llist_abort_desc(wq, ie, desc);

			return rc;

		}



	percpu_ref_put(&wq->wq_active);



	/*

	 * Pending the descriptor to the lockless list for the irq_entry

	 * that we designated the descriptor to.

	 */

	if (desc->hw->flags & IDXD_OP_FLAG_RCI) {

		int vec;



		/*

		 * If the driver is on host kernel, it would be the value

		 * assigned to interrupt handle, which is index for MSIX

		 * vector. If it's guest then can't use the int_handle since

		 * that is the index to IMS for the entire device. The guest

		 * device local index will be used.

		 */

		vec = !idxd->int_handles ? desc->hw->int_handle : desc->vector;

		llist_add(&desc->llnode, &idxd->irq_entries[vec].pending_llist);

	}



	percpu_ref_put(&wq->wq_active);

	return 0;

}

drivers/dma/idxd/sysfs.c

+0 −2
Original line number Diff line number Diff line
@@ -1744,8 +1744,6 @@ void idxd_unregister_devices(struct idxd_device *idxd) 


		device_unregister(&group->conf_dev);

	}



	device_unregister(&idxd->conf_dev);

}



int idxd_register_bus_type(void)