Unverified Commit c7361854 authored by Mark Brown's avatar Mark Brown
Browse files

Support ECSPI dynamic burst feature for DMA mode 

Merge series from Carlos Song <carlos.song@nxp.com>:

ECSPI has a low throughput because of no dynamic burst support, it
transfers only one word per frame in DMA mode, causing SCLK stalls
between words due to BURST_LENGTH updates.

This patch set is to support ECSPI dynamic burst feature to help improve
the ECSPI DMA mode performance.

Performance test (spidev_test @10MHz, 4KB):
  Before: tx/rx ~6651.9 kbps
  After:  tx/rx ~9922.2 kbps (~50% improvement)

For compatibility with slow SPI devices, add configurable word delay in
DMA mode. When word delay is set, dynamic burst is disabled and
BURST_LENGTH equals word length.

Also support target DMA mode with enabled dynamic burst.

---
Changes since v2:
* Patch1~4: no change
* Patch5: remove __counted_by from *dma_data, because spi_imx struct
  is allocated with a fixed memory in probe, flexible array is not
  suitable, so just remove it to avoid the build error. Also add
  review-by tag
* Patch6: add review-by tag
Changes since v1:
* Patch1~3:
* Add review-by tag
* Patch4:
  * Add review-by tag
  * Improve patch commit log
* Patch5:
  * Clean up tab instead of space for dma_tx_addr and dma_rx_addr
  * Add  __counted_by(dma_package_num) for dma_data_package *dma_data
  * Move temp and bytes_per_word to if()
  * Add comment for spi_imx_dma_rx_data_handle() and
    spi_imx_dma_tx_data_handle()
  * Use kzalloc instead of kmalloc(x, x | __GFP_ZERO)
  * Use sizeof(*temp) instead of BYTES_PER_32BITS_WORD in suitable place
  * Add comments and fix comments according to discussion
  * Propagate the error value of dma_mapping_error()
  * Use round_down() to replace DIV_ROUND_DOWN_ULL()
  * Improve transfer->error |= SPI_TRANS_FAIL_NO_START judgment for
    better readability
* Patch6:
  * Improve patch commit log
  * Add READ_ONCE for spi_imx->target_aborted
  * Add comments for data length limit
  * Move data length limit to spi_imx_can_dma()

Carlos Song (6):
  spi: imx: group spi_imx_dma_configure() with spi_imx_dma_transfer()
  spi: imx: introduce helper to clear DMA mode logic
  spi: imx: avoid dmaengine_terminate_all() on TX prep failure
  spi: imx: handle DMA submission errors with dma_submit_error()
  spi: imx: support dynamic burst length for ECSPI DMA mode
  spi: imx: enable DMA mode for target operation

 drivers/spi/spi-imx.c | 625 +++++++++++++++++++++++++++++++++++-------
 1 file changed, 519 insertions(+), 106 deletions(-)

--
2.34.1
parents 403a4f3c ba9b2865

drivers/spi/spi-imx.c

+519 −106
Original line number Diff line number Diff line
@@ -60,6 +60,7 @@ MODULE_PARM_DESC(polling_limit_us, 
#define MX51_ECSPI_CTRL_MAX_BURST	512

/* The maximum bytes that IMX53_ECSPI can transfer in target mode.*/

#define MX53_MAX_TRANSFER_BYTES		512

#define BYTES_PER_32BITS_WORD		4



enum spi_imx_devtype {

	IMX1_CSPI,
@@ -95,6 +96,16 @@ struct spi_imx_devtype_data { 
	enum spi_imx_devtype devtype;

};



struct dma_data_package {

	u32 cmd_word;

	void *dma_rx_buf;

	void *dma_tx_buf;

	dma_addr_t dma_tx_addr;

	dma_addr_t dma_rx_addr;

	int dma_len;

	int data_len;

};



struct spi_imx_data {

	struct spi_controller *controller;

	struct device *dev;
@@ -130,6 +141,9 @@ struct spi_imx_data { 
	u32 wml;

	struct completion dma_rx_completion;

	struct completion dma_tx_completion;

	size_t dma_package_num;

	struct dma_data_package *dma_data;

	int rx_offset;



	const struct spi_imx_devtype_data *devtype_data;

};
@@ -189,6 +203,9 @@ MXC_SPI_BUF_TX(u16) 
MXC_SPI_BUF_RX(u32)

MXC_SPI_BUF_TX(u32)



/* Align to cache line to avoid swiotlo bounce */

#define DMA_CACHE_ALIGNED_LEN(x) ALIGN((x), dma_get_cache_alignment())



/* First entry is reserved, second entry is valid only if SDHC_SPIEN is set

 * (which is currently not the case in this driver)

 */
@@ -247,12 +264,26 @@ static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device 
	if (!controller->dma_rx)

		return false;



	if (spi_imx->target_mode)

	/*

	 * Due to Freescale errata ERR003775 "eCSPI: Burst completion by Chip

	 * Select (SS) signal in Slave mode is not functional" burst size must

	 * be set exactly to the size of the transfer. This limit SPI transaction

	 * with maximum 2^12 bits.

	 */

	if (transfer->len > MX53_MAX_TRANSFER_BYTES && spi_imx->target_mode)

		return false;



	if (transfer->len < spi_imx->devtype_data->fifo_size)

		return false;



	/* DMA only can transmit data in bytes */

	if (spi_imx->bits_per_word != 8 && spi_imx->bits_per_word != 16 &&

	    spi_imx->bits_per_word != 32)

		return false;



	if (transfer->len >= MAX_SDMA_BD_BYTES)

		return false;



	spi_imx->dynamic_burst = 0;



	return true;
@@ -1282,50 +1313,6 @@ static irqreturn_t spi_imx_isr(int irq, void *dev_id) 
	return IRQ_HANDLED;

}



static int spi_imx_dma_configure(struct spi_controller *controller)

{

	int ret;

	enum dma_slave_buswidth buswidth;

	struct dma_slave_config rx = {}, tx = {};

	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);



	switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {

	case 4:

		buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;

		break;

	case 2:

		buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;

		break;

	case 1:

		buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;

		break;

	default:

		return -EINVAL;

	}



	tx.direction = DMA_MEM_TO_DEV;

	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;

	tx.dst_addr_width = buswidth;

	tx.dst_maxburst = spi_imx->wml;

	ret = dmaengine_slave_config(controller->dma_tx, &tx);

	if (ret) {

		dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);

		return ret;

	}



	rx.direction = DMA_DEV_TO_MEM;

	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;

	rx.src_addr_width = buswidth;

	rx.src_maxburst = spi_imx->wml;

	ret = dmaengine_slave_config(controller->dma_rx, &rx);

	if (ret) {

		dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);

		return ret;

	}



	return 0;

}



static int spi_imx_setupxfer(struct spi_device *spi,

				 struct spi_transfer *t)

{
@@ -1442,8 +1429,6 @@ static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx, 


	init_completion(&spi_imx->dma_rx_completion);

	init_completion(&spi_imx->dma_tx_completion);

	controller->can_dma = spi_imx_can_dma;

	controller->max_dma_len = MAX_SDMA_BD_BYTES;

	spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |

					 SPI_CONTROLLER_MUST_TX;
@@ -1481,71 +1466,302 @@ static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size) 
	return secs_to_jiffies(2 * timeout);

}



static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,

				struct spi_transfer *transfer)

static void spi_imx_dma_unmap(struct spi_imx_data *spi_imx,

			      struct dma_data_package *dma_data)

{

	struct device *tx_dev = spi_imx->controller->dma_tx->device->dev;

	struct device *rx_dev = spi_imx->controller->dma_rx->device->dev;



	dma_unmap_single(tx_dev, dma_data->dma_tx_addr,

			 DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),

			 DMA_TO_DEVICE);

	dma_unmap_single(rx_dev, dma_data->dma_rx_addr,

			 DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),

			 DMA_FROM_DEVICE);

}



static void spi_imx_dma_rx_data_handle(struct spi_imx_data *spi_imx,

				       struct dma_data_package *dma_data, void *rx_buf,

				       bool word_delay)

{

	void *copy_ptr;

	int unaligned;



	/*

	 * On little-endian CPUs, adjust byte order:

	 * - Swap bytes when bpw = 8

	 * - Swap half-words when bpw = 16

	 * This ensures correct data ordering for DMA transfers.

	 */

#ifdef __LITTLE_ENDIAN

	if (!word_delay) {

		unsigned int bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);

		u32 *temp = dma_data->dma_rx_buf;



		for (int i = 0; i < DIV_ROUND_UP(dma_data->dma_len, sizeof(*temp)); i++) {

			if (bytes_per_word == 1)

				swab32s(temp + i);

			else if (bytes_per_word == 2)

				swahw32s(temp + i);

		}

	}

#endif



	/*

	 * When dynamic burst enabled, DMA RX always receives 32-bit words from RXFIFO with

	 * buswidth = 4, but when data_len is not 4-bytes alignment, the RM shows when

	 * burst length = 32*n + m bits, a SPI burst contains the m LSB in first word and all

	 * 32 bits in other n words. So if garbage bytes in the first word, trim first word then

	 * copy the actual data to rx_buf.

	 */

	if (dma_data->data_len % BYTES_PER_32BITS_WORD && !word_delay) {

		unaligned = dma_data->data_len % BYTES_PER_32BITS_WORD;

		copy_ptr = (u8 *)dma_data->dma_rx_buf + BYTES_PER_32BITS_WORD - unaligned;

	} else {

		copy_ptr = dma_data->dma_rx_buf;

	}



	memcpy(rx_buf, copy_ptr, dma_data->data_len);

}



static int spi_imx_dma_map(struct spi_imx_data *spi_imx,

			   struct dma_data_package *dma_data)

{

	struct dma_async_tx_descriptor *desc_tx, *desc_rx;

	unsigned long transfer_timeout;

	unsigned long time_left;

	struct spi_controller *controller = spi_imx->controller;

	struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;

	struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents);

	unsigned int bytes_per_word, i;

	struct device *tx_dev = controller->dma_tx->device->dev;

	struct device *rx_dev = controller->dma_rx->device->dev;

	int ret;



	/* Get the right burst length from the last sg to ensure no tail data */

	bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);

	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {

		if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))

			break;

	dma_data->dma_tx_addr = dma_map_single(tx_dev, dma_data->dma_tx_buf,

					       DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),

					       DMA_TO_DEVICE);

	ret = dma_mapping_error(tx_dev, dma_data->dma_tx_addr);

	if (ret < 0) {

		dev_err(spi_imx->dev, "DMA TX map failed %d\n", ret);

		return ret;

	}

	/* Use 1 as wml in case no available burst length got */

	if (i == 0)

		i = 1;



	spi_imx->wml =  i;

	dma_data->dma_rx_addr = dma_map_single(rx_dev, dma_data->dma_rx_buf,

					       DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),

					       DMA_FROM_DEVICE);

	ret = dma_mapping_error(rx_dev, dma_data->dma_rx_addr);

	if (ret < 0) {

		dev_err(spi_imx->dev, "DMA RX map failed %d\n", ret);

		dma_unmap_single(tx_dev, dma_data->dma_tx_addr,

				 DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),

				 DMA_TO_DEVICE);

		return ret;

	}



	ret = spi_imx_dma_configure(controller);

	if (ret)

		goto dma_failure_no_start;

	return 0;

}



	if (!spi_imx->devtype_data->setup_wml) {

		dev_err(spi_imx->dev, "No setup_wml()?\n");

		ret = -EINVAL;

		goto dma_failure_no_start;

static int spi_imx_dma_tx_data_handle(struct spi_imx_data *spi_imx,

				      struct dma_data_package *dma_data,

				      const void *tx_buf,

				      bool word_delay)

{

	void *copy_ptr;

	int unaligned;



	if (word_delay) {

		dma_data->dma_len = dma_data->data_len;

	} else {

		/*

		 * As per the reference manual, when burst length = 32*n + m bits, ECSPI

		 * sends m LSB bits in the first word, followed by n full 32-bit words.

		 * Since actual data may not be 4-byte aligned, allocate DMA TX/RX buffers

		 * to ensure alignment. For TX, DMA pushes 4-byte aligned words to TXFIFO,

		 * while ECSPI uses BURST_LENGTH settings to maintain correct bit count.

		 * For RX, DMA always receives 32-bit words from RXFIFO, when data len is

		 * not 4-byte aligned, trim the first word to drop garbage bytes, then group

		 * all transfer DMA bounse buffer and copy all valid data to rx_buf.

		 */

		dma_data->dma_len = ALIGN(dma_data->data_len, BYTES_PER_32BITS_WORD);

	}



	dma_data->dma_tx_buf = kzalloc(dma_data->dma_len, GFP_KERNEL);

	if (!dma_data->dma_tx_buf)

		return -ENOMEM;



	dma_data->dma_rx_buf = kzalloc(dma_data->dma_len, GFP_KERNEL);

	if (!dma_data->dma_rx_buf) {

		kfree(dma_data->dma_tx_buf);

		return -ENOMEM;

	}



	if (dma_data->data_len % BYTES_PER_32BITS_WORD && !word_delay) {

		unaligned = dma_data->data_len % BYTES_PER_32BITS_WORD;

		copy_ptr = (u8 *)dma_data->dma_tx_buf + BYTES_PER_32BITS_WORD - unaligned;

	} else {

		copy_ptr = dma_data->dma_tx_buf;

	}



	memcpy(copy_ptr, tx_buf, dma_data->data_len);



	/*

	 * When word_delay is enabled, DMA transfers an entire word in one minor loop.

	 * In this case, no data requires additional handling.

	 */

	if (word_delay)

		return 0;



#ifdef __LITTLE_ENDIAN

	/*

	 * On little-endian CPUs, adjust byte order:

	 * - Swap bytes when bpw = 8

	 * - Swap half-words when bpw = 16

	 * This ensures correct data ordering for DMA transfers.

	 */

	unsigned int bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);

	u32 *temp = dma_data->dma_tx_buf;



	for (int i = 0; i < DIV_ROUND_UP(dma_data->dma_len, sizeof(*temp)); i++) {

		if (bytes_per_word == 1)

			swab32s(temp + i);

		else if (bytes_per_word == 2)

			swahw32s(temp + i);

	}

#endif



	return 0;

}



static int spi_imx_dma_data_prepare(struct spi_imx_data *spi_imx,

				    struct spi_transfer *transfer,

				    bool word_delay)

{

	u32 pre_bl, tail_bl;

	u32 ctrl;

	int ret;



	/*

	 * ECSPI supports a maximum burst of 512 bytes. When xfer->len exceeds 512

	 * and is not a multiple of 512, a tail transfer is required. BURST_LEGTH

	 * is used for SPI HW to maintain correct bit count. BURST_LENGTH should

	 * update with data length. After DMA request submit, SPI can not update the

	 * BURST_LENGTH, in this case, we must split two package, update the register

	 * then setup second DMA transfer.

	 */

	ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);

	if (word_delay) {

		/*

		 * When SPI IMX need to support word delay, according to "Sample Period Control

		 * Register" shows, The Sample Period Control Register (ECSPI_PERIODREG)

		 * provides software a way to insert delays (wait states) between consecutive

		 * SPI transfers. As a result, ECSPI can only transfer one word per frame, and

		 * the delay occurs between frames.

		 */

		spi_imx->dma_package_num = 1;

		pre_bl = spi_imx->bits_per_word - 1;

	} else if (transfer->len <= MX51_ECSPI_CTRL_MAX_BURST) {

		spi_imx->dma_package_num = 1;

		pre_bl = transfer->len * BITS_PER_BYTE - 1;

	} else if (!(transfer->len % MX51_ECSPI_CTRL_MAX_BURST)) {

		spi_imx->dma_package_num = 1;

		pre_bl = MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1;

	} else {

		spi_imx->dma_package_num = 2;

		pre_bl = MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1;

		tail_bl = (transfer->len % MX51_ECSPI_CTRL_MAX_BURST) * BITS_PER_BYTE - 1;

	}



	spi_imx->dma_data = kmalloc_array(spi_imx->dma_package_num,

					  sizeof(struct dma_data_package),

					  GFP_KERNEL | __GFP_ZERO);

	if (!spi_imx->dma_data) {

		dev_err(spi_imx->dev, "Failed to allocate DMA package buffer!\n");

		return -ENOMEM;

	}



	if (spi_imx->dma_package_num == 1) {

		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;

		ctrl |= pre_bl << MX51_ECSPI_CTRL_BL_OFFSET;

		spi_imx->dma_data[0].cmd_word = ctrl;

		spi_imx->dma_data[0].data_len = transfer->len;

		ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[0], transfer->tx_buf,

						 word_delay);

		if (ret) {

			kfree(spi_imx->dma_data);

			return ret;

		}

	} else {

		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;

		ctrl |= pre_bl << MX51_ECSPI_CTRL_BL_OFFSET;

		spi_imx->dma_data[0].cmd_word = ctrl;

		spi_imx->dma_data[0].data_len = round_down(transfer->len,

							   MX51_ECSPI_CTRL_MAX_BURST);

		ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[0], transfer->tx_buf,

						 false);

		if (ret) {

			kfree(spi_imx->dma_data);

			return ret;

		}



		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;

		ctrl |= tail_bl << MX51_ECSPI_CTRL_BL_OFFSET;

		spi_imx->dma_data[1].cmd_word = ctrl;

		spi_imx->dma_data[1].data_len = transfer->len % MX51_ECSPI_CTRL_MAX_BURST;

		ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[1],

						 transfer->tx_buf + spi_imx->dma_data[0].data_len,

						 false);

		if (ret) {

			kfree(spi_imx->dma_data[0].dma_tx_buf);

			kfree(spi_imx->dma_data[0].dma_rx_buf);

			kfree(spi_imx->dma_data);

		}

	}

	spi_imx->devtype_data->setup_wml(spi_imx);



	return 0;

}



static int spi_imx_dma_submit(struct spi_imx_data *spi_imx,

			      struct dma_data_package *dma_data,

			      struct spi_transfer *transfer)

{

	struct spi_controller *controller = spi_imx->controller;

	struct dma_async_tx_descriptor *desc_tx, *desc_rx;

	unsigned long transfer_timeout;

	unsigned long time_left;

	dma_cookie_t cookie;



	/*

	 * The TX DMA setup starts the transfer, so make sure RX is configured

	 * before TX.

	 */

	desc_rx = dmaengine_prep_slave_sg(controller->dma_rx,

				rx->sgl, rx->nents, DMA_DEV_TO_MEM,

	desc_rx = dmaengine_prep_slave_single(controller->dma_rx, dma_data->dma_rx_addr,

					      dma_data->dma_len, DMA_DEV_TO_MEM,

					      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);

	if (!desc_rx) {

		ret = -EINVAL;

		goto dma_failure_no_start;

		transfer->error |= SPI_TRANS_FAIL_NO_START;

		return -EINVAL;

	}



	desc_rx->callback = spi_imx_dma_rx_callback;

	desc_rx->callback_param = (void *)spi_imx;

	dmaengine_submit(desc_rx);

	cookie = dmaengine_submit(desc_rx);

	if (dma_submit_error(cookie)) {

		dev_err(spi_imx->dev, "submitting DMA RX failed\n");

		transfer->error |= SPI_TRANS_FAIL_NO_START;

		goto dmaengine_terminate_rx;

	}



	reinit_completion(&spi_imx->dma_rx_completion);

	dma_async_issue_pending(controller->dma_rx);



	desc_tx = dmaengine_prep_slave_sg(controller->dma_tx,

				tx->sgl, tx->nents, DMA_MEM_TO_DEV,

	desc_tx = dmaengine_prep_slave_single(controller->dma_tx, dma_data->dma_tx_addr,

					      dma_data->dma_len, DMA_MEM_TO_DEV,

					      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);

	if (!desc_tx) {

		dmaengine_terminate_all(controller->dma_tx);

		dmaengine_terminate_all(controller->dma_rx);

		return -EINVAL;

	}

	if (!desc_tx)

		goto dmaengine_terminate_rx;



	desc_tx->callback = spi_imx_dma_tx_callback;

	desc_tx->callback_param = (void *)spi_imx;

	dmaengine_submit(desc_tx);

	cookie = dmaengine_submit(desc_tx);

	if (dma_submit_error(cookie)) {

		dev_err(spi_imx->dev, "submitting DMA TX failed\n");

		goto dmaengine_terminate_tx;

	}

	reinit_completion(&spi_imx->dma_tx_completion);

	dma_async_issue_pending(controller->dma_tx);
@@ -1553,6 +1769,7 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx, 


	transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);



	if (!spi_imx->target_mode) {

		/* Wait SDMA to finish the data transfer.*/

		time_left = wait_for_completion_timeout(&spi_imx->dma_tx_completion,

							transfer_timeout);
@@ -1571,6 +1788,143 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx, 
			dmaengine_terminate_all(controller->dma_rx);

			return -ETIMEDOUT;

		}

	} else {

		spi_imx->target_aborted = false;



		if (wait_for_completion_interruptible(&spi_imx->dma_tx_completion) ||

		    READ_ONCE(spi_imx->target_aborted)) {

			dev_dbg(spi_imx->dev, "I/O Error in DMA TX interrupted\n");

			dmaengine_terminate_all(controller->dma_tx);

			dmaengine_terminate_all(controller->dma_rx);

			return -EINTR;

		}



		if (wait_for_completion_interruptible(&spi_imx->dma_rx_completion) ||

		    READ_ONCE(spi_imx->target_aborted)) {

			dev_dbg(spi_imx->dev, "I/O Error in DMA RX interrupted\n");

			dmaengine_terminate_all(controller->dma_rx);

			return -EINTR;

		}



		/*

		 * ECSPI has a HW issue when works in Target mode, after 64 words

		 * writtern to TXFIFO, even TXFIFO becomes empty, ECSPI_TXDATA keeps

		 * shift out the last word data, so we have to disable ECSPI when in

		 * target mode after the transfer completes.

		 */

		if (spi_imx->devtype_data->disable)

			spi_imx->devtype_data->disable(spi_imx);

	}



	return 0;



dmaengine_terminate_tx:

	dmaengine_terminate_all(controller->dma_tx);

dmaengine_terminate_rx:

	dmaengine_terminate_all(controller->dma_rx);



	return -EINVAL;

}



static void spi_imx_dma_max_wml_find(struct spi_imx_data *spi_imx,

				     struct dma_data_package *dma_data,

				     bool word_delay)

{

	unsigned int bytes_per_word = word_delay ?

				      spi_imx_bytes_per_word(spi_imx->bits_per_word) :

				      BYTES_PER_32BITS_WORD;

	unsigned int i;



	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {

		if (!dma_data->dma_len % (i * bytes_per_word))

			break;

	}

	/* Use 1 as wml in case no available burst length got */

	if (i == 0)

		i = 1;



	spi_imx->wml = i;

}



static int spi_imx_dma_configure(struct spi_controller *controller, bool word_delay)

{

	int ret;

	enum dma_slave_buswidth buswidth;

	struct dma_slave_config rx = {}, tx = {};

	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);



	if (word_delay) {

		switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {

		case 4:

			buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;

			break;

		case 2:

			buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;

			break;

		case 1:

			buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;

			break;

		default:

			return -EINVAL;

		}

	} else {

		buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;

	}



	tx.direction = DMA_MEM_TO_DEV;

	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;

	tx.dst_addr_width = buswidth;

	tx.dst_maxburst = spi_imx->wml;

	ret = dmaengine_slave_config(controller->dma_tx, &tx);

	if (ret) {

		dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);

		return ret;

	}



	rx.direction = DMA_DEV_TO_MEM;

	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;

	rx.src_addr_width = buswidth;

	rx.src_maxburst = spi_imx->wml;

	ret = dmaengine_slave_config(controller->dma_rx, &rx);

	if (ret) {

		dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);

		return ret;

	}



	return 0;

}



static int spi_imx_dma_package_transfer(struct spi_imx_data *spi_imx,

					struct dma_data_package *dma_data,

					struct spi_transfer *transfer,

					bool word_delay)

{

	struct spi_controller *controller = spi_imx->controller;

	int ret;



	spi_imx_dma_max_wml_find(spi_imx, dma_data, word_delay);



	ret = spi_imx_dma_configure(controller, word_delay);

	if (ret)

		goto dma_failure_no_start;



	if (!spi_imx->devtype_data->setup_wml) {

		dev_err(spi_imx->dev, "No setup_wml()?\n");

		ret = -EINVAL;

		goto dma_failure_no_start;

	}

	spi_imx->devtype_data->setup_wml(spi_imx);



	ret = spi_imx_dma_submit(spi_imx, dma_data, transfer);

	if (ret)

		return ret;



	/* Trim the DMA RX buffer and copy the actual data to rx_buf */

	dma_sync_single_for_cpu(controller->dma_rx->device->dev, dma_data->dma_rx_addr,

				dma_data->dma_len, DMA_FROM_DEVICE);

	spi_imx_dma_rx_data_handle(spi_imx, dma_data, transfer->rx_buf + spi_imx->rx_offset,

				   word_delay);

	spi_imx->rx_offset += dma_data->data_len;



	return 0;

/* fallback to pio */
@@ -1579,6 +1933,57 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx, 
	return ret;

}



static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,

				struct spi_transfer *transfer)

{

	bool word_delay = transfer->word_delay.value != 0 && !spi_imx->target_mode;

	int ret;

	int i;



	ret = spi_imx_dma_data_prepare(spi_imx, transfer, word_delay);

	if (ret < 0) {

		transfer->error |= SPI_TRANS_FAIL_NO_START;

		dev_err(spi_imx->dev, "DMA data prepare fail\n");

		goto fallback_pio;

	}



	spi_imx->rx_offset = 0;



	/* Each dma_package performs a separate DMA transfer once */

	for (i = 0; i < spi_imx->dma_package_num; i++) {

		ret = spi_imx_dma_map(spi_imx, &spi_imx->dma_data[i]);

		if (ret < 0) {

			if (i == 0)

				transfer->error |= SPI_TRANS_FAIL_NO_START;

			dev_err(spi_imx->dev, "DMA map fail\n");

			break;

		}



		/* Update the CTRL register BL field */

		writel(spi_imx->dma_data[i].cmd_word, spi_imx->base + MX51_ECSPI_CTRL);



		ret = spi_imx_dma_package_transfer(spi_imx, &spi_imx->dma_data[i],

						   transfer, word_delay);



		/* Whether the dma transmission is successful or not, dma unmap is necessary */

		spi_imx_dma_unmap(spi_imx, &spi_imx->dma_data[i]);



		if (ret < 0) {

			dev_dbg(spi_imx->dev, "DMA %d transfer not really finish\n", i);

			break;

		}

	}



	for (int j = 0; j < spi_imx->dma_package_num; j++) {

		kfree(spi_imx->dma_data[j].dma_tx_buf);

		kfree(spi_imx->dma_data[j].dma_rx_buf);

	}

	kfree(spi_imx->dma_data);



fallback_pio:

	return ret;

}



static int spi_imx_pio_transfer(struct spi_device *spi,

				struct spi_transfer *transfer)

{
@@ -1737,7 +2142,7 @@ static int spi_imx_transfer_one(struct spi_controller *controller, 
	while (spi_imx->devtype_data->rx_available(spi_imx))

		readl(spi_imx->base + MXC_CSPIRXDATA);



	if (spi_imx->target_mode)

	if (spi_imx->target_mode && !spi_imx->usedma)

		return spi_imx_pio_transfer_target(spi, transfer);



	/*
@@ -1745,9 +2150,17 @@ static int spi_imx_transfer_one(struct spi_controller *controller, 
	 * transfer, the SPI transfer has already been mapped, so we

	 * have to do the DMA transfer here.

	 */

	if (spi_imx->usedma)

		return spi_imx_dma_transfer(spi_imx, transfer);



	if (spi_imx->usedma) {

		ret = spi_imx_dma_transfer(spi_imx, transfer);

		if (transfer->error & SPI_TRANS_FAIL_NO_START) {

			spi_imx->usedma = false;

			if (spi_imx->target_mode)

				return spi_imx_pio_transfer_target(spi, transfer);

			else

				return spi_imx_pio_transfer(spi, transfer);

		}

		return ret;

	}

	/* run in polling mode for short transfers */

	if (transfer->len == 1 || (polling_limit_us &&

				   spi_imx_transfer_estimate_time_us(transfer) < polling_limit_us))