Commit 8756ef2e authored by Meghana Malladi's avatar Meghana Malladi Committed by Paolo Abeni
Browse files

net: ti: icssg-prueth: Add AF_XDP zero copy for TX 



Use xsk_pool inside tx_chn to check if a given Tx queue id
is registered for xsk zero copy, which gets populated during
xsk enable

If xsk_pool is set, get frames from the pool in NAPI
context and submit them to the Tx channel. Tx completion
is also handled in the NAPI context.

Use PRUETH_SWDATA_XSK to recycle xsk buffers back to the
umem pool. Add XDP_WAKEUP_TX support to enable xsk_wakeup
for Tx.

Reviewed-by: default avatarJacob Keller <jacob.e.keller@intel.com>
Signed-off-by: default avatarMeghana Malladi <m-malladi@ti.com>
Link: https://patch.msgid.link/20251118135542.380574-4-m-malladi@ti.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent 7dfd7597

drivers/net/ethernet/ti/icssg/icssg_common.c

+110 −2
Original line number Diff line number Diff line
@@ -93,15 +93,91 @@ void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num) 
}

EXPORT_SYMBOL_GPL(prueth_ndev_del_tx_napi);



static int emac_xsk_xmit_zc(struct prueth_emac *emac,

			    unsigned int q_idx)

{

	struct prueth_tx_chn *tx_chn = &emac->tx_chns[q_idx];

	struct xsk_buff_pool *pool = tx_chn->xsk_pool;

	struct net_device *ndev = emac->ndev;

	struct cppi5_host_desc_t *host_desc;

	dma_addr_t dma_desc, dma_buf;

	struct prueth_swdata *swdata;

	struct xdp_desc xdp_desc;

	int num_tx = 0, pkt_len;

	int descs_avail, ret;

	u32 *epib;

	int i;



	descs_avail = k3_cppi_desc_pool_avail(tx_chn->desc_pool);

	/* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS

	 * will be available for normal TX path and queue is stopped there if

	 * necessary

	 */

	if (descs_avail <= MAX_SKB_FRAGS)

		return 0;



	descs_avail -= MAX_SKB_FRAGS;



	for (i = 0; i < descs_avail; i++) {

		if (!xsk_tx_peek_desc(pool, &xdp_desc))

			break;



		dma_buf = xsk_buff_raw_get_dma(pool, xdp_desc.addr);

		pkt_len = xdp_desc.len;

		xsk_buff_raw_dma_sync_for_device(pool, dma_buf, pkt_len);



		host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);

		if (unlikely(!host_desc))

			break;



		cppi5_hdesc_init(host_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,

				 PRUETH_NAV_PS_DATA_SIZE);

		cppi5_hdesc_set_pkttype(host_desc, 0);

		epib = host_desc->epib;

		epib[0] = 0;

		epib[1] = 0;

		cppi5_hdesc_set_pktlen(host_desc, pkt_len);

		cppi5_desc_set_tags_ids(&host_desc->hdr, 0,

					(emac->port_id | (q_idx << 8)));



		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf);

		cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf,

				       pkt_len);



		swdata = cppi5_hdesc_get_swdata(host_desc);

		swdata->type = PRUETH_SWDATA_XSK;



		dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,

						      host_desc);

		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn,

					       host_desc, dma_desc);



		if (ret) {

			ndev->stats.tx_errors++;

			k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc);

			break;

		}



		num_tx++;

	}



	xsk_tx_release(tx_chn->xsk_pool);

	return num_tx;

}



void prueth_xmit_free(struct prueth_tx_chn *tx_chn,

		      struct cppi5_host_desc_t *desc)

{

	struct cppi5_host_desc_t *first_desc, *next_desc;

	dma_addr_t buf_dma, next_desc_dma;

	struct prueth_swdata *swdata;

	u32 buf_dma_len;



	first_desc = desc;

	next_desc = first_desc;

	swdata = cppi5_hdesc_get_swdata(first_desc);

	if (swdata->type == PRUETH_SWDATA_XSK)

		goto free_pool;



	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);

	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
@@ -126,6 +202,7 @@ void prueth_xmit_free(struct prueth_tx_chn *tx_chn, 
		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);

	}



free_pool:

	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);

}

EXPORT_SYMBOL_GPL(prueth_xmit_free);
@@ -139,7 +216,9 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 
	struct prueth_swdata *swdata;

	struct prueth_tx_chn *tx_chn;

	unsigned int total_bytes = 0;

	int xsk_frames_done = 0;

	struct xdp_frame *xdpf;

	unsigned int pkt_len;

	struct sk_buff *skb;

	dma_addr_t desc_dma;

	int res, num_tx = 0;
@@ -176,6 +255,11 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 
			total_bytes += xdpf->len;

			xdp_return_frame(xdpf);

			break;

		case PRUETH_SWDATA_XSK:

			pkt_len = cppi5_hdesc_get_pktlen(desc_tx);

			dev_sw_netstats_tx_add(ndev, 1, pkt_len);

			xsk_frames_done++;

			break;

		default:

			prueth_xmit_free(tx_chn, desc_tx);

			ndev->stats.tx_dropped++;
@@ -204,6 +288,18 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 
		__netif_tx_unlock(netif_txq);

	}



	if (tx_chn->xsk_pool) {

		if (xsk_frames_done)

			xsk_tx_completed(tx_chn->xsk_pool, xsk_frames_done);



		if (xsk_uses_need_wakeup(tx_chn->xsk_pool))

			xsk_set_tx_need_wakeup(tx_chn->xsk_pool);



		netif_txq = netdev_get_tx_queue(ndev, chn);

		txq_trans_cond_update(netif_txq);

		emac_xsk_xmit_zc(emac, chn);

	}



	return num_tx;

}
@@ -212,7 +308,10 @@ static enum hrtimer_restart emac_tx_timer_callback(struct hrtimer *timer) 
	struct prueth_tx_chn *tx_chns =

			container_of(timer, struct prueth_tx_chn, tx_hrtimer);



	if (tx_chns->irq_disabled) {

		tx_chns->irq_disabled = false;

		enable_irq(tx_chns->irq);

	}

	return HRTIMER_NORESTART;

}
@@ -235,9 +334,12 @@ static int emac_napi_tx_poll(struct napi_struct *napi_tx, int budget) 
				      ns_to_ktime(tx_chn->tx_pace_timeout_ns),

				      HRTIMER_MODE_REL_PINNED);

		} else {

			if (tx_chn->irq_disabled) {

				tx_chn->irq_disabled = false;

				enable_irq(tx_chn->irq);

			}

		}

	}



	return num_tx_packets;

}
@@ -246,6 +348,7 @@ static irqreturn_t prueth_tx_irq(int irq, void *dev_id) 
{

	struct prueth_tx_chn *tx_chn = dev_id;



	tx_chn->irq_disabled = true;

	disable_irq_nosync(irq);

	napi_schedule(&tx_chn->napi_tx);
@@ -1032,6 +1135,7 @@ void prueth_tx_cleanup(void *data, dma_addr_t desc_dma) 
{

	struct prueth_tx_chn *tx_chn = data;

	struct cppi5_host_desc_t *desc_tx;

	struct xsk_buff_pool *xsk_pool;

	struct prueth_swdata *swdata;

	struct xdp_frame *xdpf;

	struct sk_buff *skb;
@@ -1048,6 +1152,10 @@ void prueth_tx_cleanup(void *data, dma_addr_t desc_dma) 
		xdpf = swdata->data.xdpf;

		xdp_return_frame(xdpf);

		break;

	case PRUETH_SWDATA_XSK:

		xsk_pool = tx_chn->xsk_pool;

		xsk_tx_completed(xsk_pool, 1);

		break;

	default:

		break;

	}

drivers/net/ethernet/ti/icssg/icssg_prueth.c

+13 −0
Original line number Diff line number Diff line
@@ -1347,12 +1347,25 @@ static int emac_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf) 
int prueth_xsk_wakeup(struct net_device *ndev, u32 qid, u32 flags)

{

	struct prueth_emac *emac = netdev_priv(ndev);

	struct prueth_tx_chn *tx_chn = &emac->tx_chns[qid];



	if (qid >= PRUETH_MAX_RX_FLOWS || qid >= emac->tx_ch_num) {

		netdev_err(ndev, "Invalid XSK queue ID %d\n", qid);

		return -EINVAL;

	}



	if (!tx_chn->xsk_pool) {

		netdev_err(ndev, "XSK pool not registered for queue %d\n", qid);

		return -EINVAL;

	}



	if (flags & XDP_WAKEUP_TX) {

		if (!napi_if_scheduled_mark_missed(&tx_chn->napi_tx)) {

			if (likely(napi_schedule_prep(&tx_chn->napi_tx)))

				__napi_schedule(&tx_chn->napi_tx);

		}

	}



	return 0;

}

drivers/net/ethernet/ti/icssg/icssg_prueth.h

+2 −0
Original line number Diff line number Diff line
@@ -129,6 +129,7 @@ struct prueth_tx_chn { 
	struct hrtimer tx_hrtimer;

	unsigned long tx_pace_timeout_ns;

	struct xsk_buff_pool *xsk_pool;

	bool irq_disabled;

};



struct prueth_rx_chn {
@@ -150,6 +151,7 @@ enum prueth_swdata_type { 
	PRUETH_SWDATA_PAGE,

	PRUETH_SWDATA_CMD,

	PRUETH_SWDATA_XDPF,

	PRUETH_SWDATA_XSK,

};



struct prueth_swdata {