Commit 6d99faf2 authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'net-ti-icssg-prueth-add-native-mode-xdp-support' 

Meghana Malladi says:

====================
net: ti: icssg-prueth: Add native mode XDP support

This series adds native XDP support using page_pool.
XDP zero copy support is not included in this patch series.

Patch 1/3: Replaces skb with page pool for Rx buffer allocation
Patch 2/3: Adds prueth_swdata struct for SWDATA for all swdata cases
Patch 3/3: Introduces native mode XDP support

v3: https://lore.kernel.org/all/20250224110102.1528552-1-m-malladi@ti.com/
====================

Link: https://patch.msgid.link/20250305101422.1908370-1-m-malladi@ti.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 40587f74 62aa3246

drivers/net/ethernet/ti/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -205,6 +205,7 @@ config TI_ICSSG_PRUETH_SR1 
	select PHYLIB

	select TI_ICSS_IEP

	select TI_K3_CPPI_DESC_POOL

	select PAGE_POOL

	depends on PRU_REMOTEPROC

	depends on NET_SWITCHDEV

	depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER

drivers/net/ethernet/ti/icssg/icssg_common.c

+339 −78
Original line number Diff line number Diff line
@@ -45,6 +45,11 @@ void prueth_cleanup_rx_chns(struct prueth_emac *emac, 
			    struct prueth_rx_chn *rx_chn,

			    int max_rflows)

{

	if (rx_chn->pg_pool) {

		page_pool_destroy(rx_chn->pg_pool);

		rx_chn->pg_pool = NULL;

	}



	if (rx_chn->desc_pool)

		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
@@ -93,11 +98,20 @@ void prueth_xmit_free(struct prueth_tx_chn *tx_chn, 
{

	struct cppi5_host_desc_t *first_desc, *next_desc;

	dma_addr_t buf_dma, next_desc_dma;

	struct prueth_swdata *swdata;

	struct page *page;

	u32 buf_dma_len;



	first_desc = desc;

	next_desc = first_desc;



	swdata = cppi5_hdesc_get_swdata(desc);

	if (swdata->type == PRUETH_SWDATA_PAGE) {

		page = swdata->data.page;

		page_pool_recycle_direct(page->pp, swdata->data.page);

		goto free_desc;

	}



	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);

	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
@@ -121,6 +135,7 @@ void prueth_xmit_free(struct prueth_tx_chn *tx_chn, 
		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);

	}



free_desc:

	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);

}

EXPORT_SYMBOL_GPL(prueth_xmit_free);
@@ -131,12 +146,13 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 
	struct net_device *ndev = emac->ndev;

	struct cppi5_host_desc_t *desc_tx;

	struct netdev_queue *netif_txq;

	struct prueth_swdata *swdata;

	struct prueth_tx_chn *tx_chn;

	unsigned int total_bytes = 0;

	struct xdp_frame *xdpf;

	struct sk_buff *skb;

	dma_addr_t desc_dma;

	int res, num_tx = 0;

	void **swdata;



	tx_chn = &emac->tx_chns[chn];
@@ -157,20 +173,27 @@ int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 
						     desc_dma);

		swdata = cppi5_hdesc_get_swdata(desc_tx);



		/* was this command's TX complete? */

		if (emac->is_sr1 && *(swdata) == emac->cmd_data) {

		switch (swdata->type) {

		case PRUETH_SWDATA_SKB:

			skb = swdata->data.skb;

			dev_sw_netstats_tx_add(skb->dev, 1, skb->len);

			total_bytes += skb->len;

			napi_consume_skb(skb, budget);

			break;

		case PRUETH_SWDATA_XDPF:

			xdpf = swdata->data.xdpf;

			dev_sw_netstats_tx_add(ndev, 1, xdpf->len);

			total_bytes += xdpf->len;

			xdp_return_frame(xdpf);

			break;

		default:

			netdev_err(ndev, "tx_complete: invalid swdata type %d\n", swdata->type);

			prueth_xmit_free(tx_chn, desc_tx);

			ndev->stats.tx_dropped++;

			continue;

		}



		skb = *(swdata);

		prueth_xmit_free(tx_chn, desc_tx);



		ndev = skb->dev;

		ndev->stats.tx_packets++;

		ndev->stats.tx_bytes += skb->len;

		total_bytes += skb->len;

		napi_consume_skb(skb, budget);

		num_tx++;

	}
@@ -461,17 +484,17 @@ int prueth_init_rx_chns(struct prueth_emac *emac, 
}

EXPORT_SYMBOL_GPL(prueth_init_rx_chns);



int prueth_dma_rx_push(struct prueth_emac *emac,

		       struct sk_buff *skb,

		       struct prueth_rx_chn *rx_chn)

int prueth_dma_rx_push_mapped(struct prueth_emac *emac,

			      struct prueth_rx_chn *rx_chn,

			      struct page *page, u32 buf_len)

{

	struct net_device *ndev = emac->ndev;

	struct cppi5_host_desc_t *desc_rx;

	u32 pkt_len = skb_tailroom(skb);

	struct prueth_swdata *swdata;

	dma_addr_t desc_dma;

	dma_addr_t buf_dma;

	void **swdata;



	buf_dma = page_pool_get_dma_addr(page) + PRUETH_HEADROOM;

	desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);

	if (!desc_rx) {

		netdev_err(ndev, "rx push: failed to allocate descriptor\n");
@@ -479,25 +502,19 @@ int prueth_dma_rx_push(struct prueth_emac *emac, 
	}

	desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);



	buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len, DMA_FROM_DEVICE);

	if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {

		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);

		netdev_err(ndev, "rx push: failed to map rx pkt buffer\n");

		return -EINVAL;

	}



	cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,

			 PRUETH_NAV_PS_DATA_SIZE);

	k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);

	cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb));

	cppi5_hdesc_attach_buf(desc_rx, buf_dma, buf_len, buf_dma, buf_len);



	swdata = cppi5_hdesc_get_swdata(desc_rx);

	*swdata = skb;

	swdata->type = PRUETH_SWDATA_PAGE;

	swdata->data.page = page;



	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0,

	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, PRUETH_RX_FLOW_DATA,

					desc_rx, desc_dma);

}

EXPORT_SYMBOL_GPL(prueth_dma_rx_push);

EXPORT_SYMBOL_GPL(prueth_dma_rx_push_mapped);



u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns)

{
@@ -535,18 +552,170 @@ void emac_rx_timestamp(struct prueth_emac *emac, 
	ssh->hwtstamp = ns_to_ktime(ns);

}



static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id)

/**

 * emac_xmit_xdp_frame - transmits an XDP frame

 * @emac: emac device

 * @xdpf: data to transmit

 * @page: page from page pool if already DMA mapped

 * @q_idx: queue id

 *

 * Return: XDP state

 */

u32 emac_xmit_xdp_frame(struct prueth_emac *emac,

			struct xdp_frame *xdpf,

			struct page *page,

			unsigned int q_idx)

{

	struct cppi5_host_desc_t *first_desc;

	struct net_device *ndev = emac->ndev;

	struct prueth_tx_chn *tx_chn;

	dma_addr_t desc_dma, buf_dma;

	struct prueth_swdata *swdata;

	u32 *epib;

	int ret;



	if (q_idx >= PRUETH_MAX_TX_QUEUES) {

		netdev_err(ndev, "xdp tx: invalid q_id %d\n", q_idx);

		return ICSSG_XDP_CONSUMED;	/* drop */

	}



	tx_chn = &emac->tx_chns[q_idx];



	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);

	if (!first_desc) {

		netdev_dbg(ndev, "xdp tx: failed to allocate descriptor\n");

		goto drop_free_descs;	/* drop */

	}



	if (page) { /* already DMA mapped by page_pool */

		buf_dma = page_pool_get_dma_addr(page);

		buf_dma += xdpf->headroom + sizeof(struct xdp_frame);

	} else { /* Map the linear buffer */

		buf_dma = dma_map_single(tx_chn->dma_dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);

		if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) {

			netdev_err(ndev, "xdp tx: failed to map data buffer\n");

			goto drop_free_descs;	/* drop */

		}

	}



	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,

			 PRUETH_NAV_PS_DATA_SIZE);

	cppi5_hdesc_set_pkttype(first_desc, 0);

	epib = first_desc->epib;

	epib[0] = 0;

	epib[1] = 0;



	/* set dst tag to indicate internal qid at the firmware which is at

	 * bit8..bit15. bit0..bit7 indicates port num for directed

	 * packets in case of switch mode operation

	 */

	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8)));

	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);

	cppi5_hdesc_attach_buf(first_desc, buf_dma, xdpf->len, buf_dma, xdpf->len);

	swdata = cppi5_hdesc_get_swdata(first_desc);

	if (page) {

		swdata->type = PRUETH_SWDATA_PAGE;

		swdata->data.page = page;

	} else {

		swdata->type = PRUETH_SWDATA_XDPF;

		swdata->data.xdpf = xdpf;

	}



	cppi5_hdesc_set_pktlen(first_desc, xdpf->len);

	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);



	ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);

	if (ret) {

		netdev_err(ndev, "xdp tx: push failed: %d\n", ret);

		goto drop_free_descs;

	}



	return ICSSG_XDP_TX;



drop_free_descs:

	prueth_xmit_free(tx_chn, first_desc);

	return ICSSG_XDP_CONSUMED;

}

EXPORT_SYMBOL_GPL(emac_xmit_xdp_frame);



/**

 * emac_run_xdp - run an XDP program

 * @emac: emac device

 * @xdp: XDP buffer containing the frame

 * @page: page with RX data if already DMA mapped

 * @len: Rx descriptor packet length

 *

 * Return: XDP state

 */

static u32 emac_run_xdp(struct prueth_emac *emac, struct xdp_buff *xdp,

			struct page *page, u32 *len)

{

	struct net_device *ndev = emac->ndev;

	struct bpf_prog *xdp_prog;

	struct xdp_frame *xdpf;

	u32 pkt_len = *len;

	u32 act, result;

	int q_idx, err;



	xdp_prog = READ_ONCE(emac->xdp_prog);

	act = bpf_prog_run_xdp(xdp_prog, xdp);

	switch (act) {

	case XDP_PASS:

		return ICSSG_XDP_PASS;

	case XDP_TX:

		/* Send packet to TX ring for immediate transmission */

		xdpf = xdp_convert_buff_to_frame(xdp);

		if (unlikely(!xdpf)) {

			ndev->stats.tx_dropped++;

			goto drop;

		}



		q_idx = smp_processor_id() % emac->tx_ch_num;

		result = emac_xmit_xdp_frame(emac, xdpf, page, q_idx);

		if (result == ICSSG_XDP_CONSUMED)

			goto drop;



		dev_sw_netstats_rx_add(ndev, xdpf->len);

		return result;

	case XDP_REDIRECT:

		err = xdp_do_redirect(emac->ndev, xdp, xdp_prog);

		if (err)

			goto drop;



		dev_sw_netstats_rx_add(ndev, pkt_len);

		return ICSSG_XDP_REDIR;

	default:

		bpf_warn_invalid_xdp_action(emac->ndev, xdp_prog, act);

		fallthrough;

	case XDP_ABORTED:

drop:

		trace_xdp_exception(emac->ndev, xdp_prog, act);

		fallthrough; /* handle aborts by dropping packet */

	case XDP_DROP:

		ndev->stats.rx_dropped++;

		page_pool_recycle_direct(emac->rx_chns.pg_pool, page);

		return ICSSG_XDP_CONSUMED;

	}

}



static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state)

{

	struct prueth_rx_chn *rx_chn = &emac->rx_chns;

	u32 buf_dma_len, pkt_len, port_id = 0;

	struct net_device *ndev = emac->ndev;

	struct cppi5_host_desc_t *desc_rx;

	struct sk_buff *skb, *new_skb;

	struct prueth_swdata *swdata;

	dma_addr_t desc_dma, buf_dma;

	void **swdata;

	struct page *page, *new_page;

	struct page_pool *pool;

	struct sk_buff *skb;

	struct xdp_buff xdp;

	u32 *psdata;

	void *pa;

	int ret;



	*xdp_state = 0;

	pool = rx_chn->pg_pool;

	ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma);

	if (ret) {

		if (ret != -ENODATA)
@@ -558,15 +727,15 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id) 
		return 0;



	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);



	swdata = cppi5_hdesc_get_swdata(desc_rx);

	skb = *swdata;



	psdata = cppi5_hdesc_get_psdata(desc_rx);

	/* RX HW timestamp */

	if (emac->rx_ts_enabled)

		emac_rx_timestamp(emac, skb, psdata);

	if (swdata->type != PRUETH_SWDATA_PAGE) {

		netdev_err(ndev, "rx_pkt: invalid swdata->type %d\n", swdata->type);

		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);

		return 0;

	}



	page = swdata->data.page;

	page_pool_dma_sync_for_cpu(pool, page, 0, PAGE_SIZE);

	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);

	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);

	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
@@ -574,32 +743,63 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id) 
	pkt_len -= 4;

	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);



	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);

	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);



	skb->dev = ndev;

	new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE);

	/* if allocation fails we drop the packet but push the

	 * descriptor back to the ring with old skb to prevent a stall

	 * descriptor back to the ring with old page to prevent a stall

	 */

	if (!new_skb) {

	new_page = page_pool_dev_alloc_pages(pool);

	if (unlikely(!new_page)) {

		new_page = page;

		ndev->stats.rx_dropped++;

		new_skb = skb;

		goto requeue;

	}



	pa = page_address(page);

	if (emac->xdp_prog) {

		xdp_init_buff(&xdp, PAGE_SIZE, &rx_chn->xdp_rxq);

		xdp_prepare_buff(&xdp, pa, PRUETH_HEADROOM, pkt_len, false);



		*xdp_state = emac_run_xdp(emac, &xdp, page, &pkt_len);

		if (*xdp_state == ICSSG_XDP_PASS)

			skb = xdp_build_skb_from_buff(&xdp);

		else

			goto requeue;

	} else {

		/* send the filled skb up the n/w stack */

		/* prepare skb and send to n/w stack */

		skb = napi_build_skb(pa, PAGE_SIZE);

	}



	if (!skb) {

		ndev->stats.rx_dropped++;

		page_pool_recycle_direct(pool, page);

		goto requeue;

	}



	skb_reserve(skb, PRUETH_HEADROOM);

	skb_put(skb, pkt_len);

	skb->dev = ndev;



	psdata = cppi5_hdesc_get_psdata(desc_rx);

	/* RX HW timestamp */

	if (emac->rx_ts_enabled)

		emac_rx_timestamp(emac, skb, psdata);



	if (emac->prueth->is_switch_mode)

		skb->offload_fwd_mark = emac->offload_fwd_mark;

	skb->protocol = eth_type_trans(skb, ndev);



	skb_mark_for_recycle(skb);

	napi_gro_receive(&emac->napi_rx, skb);

	ndev->stats.rx_bytes += pkt_len;

	ndev->stats.rx_packets++;

	}



requeue:

	/* queue another RX DMA */

	ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_chns);

	ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page,

					PRUETH_MAX_PKT_SIZE);

	if (WARN_ON(ret < 0)) {

		dev_kfree_skb_any(new_skb);

		page_pool_recycle_direct(pool, new_page);

		ndev->stats.rx_errors++;

		ndev->stats.rx_dropped++;

	}
@@ -611,22 +811,19 @@ static void prueth_rx_cleanup(void *data, dma_addr_t desc_dma) 
{

	struct prueth_rx_chn *rx_chn = data;

	struct cppi5_host_desc_t *desc_rx;

	struct sk_buff *skb;

	dma_addr_t buf_dma;

	u32 buf_dma_len;

	void **swdata;

	struct prueth_swdata *swdata;

	struct page_pool *pool;

	struct page *page;



	pool = rx_chn->pg_pool;

	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);

	swdata = cppi5_hdesc_get_swdata(desc_rx);

	skb = *swdata;

	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);

	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);

	if (swdata->type == PRUETH_SWDATA_PAGE) {

		page = swdata->data.page;

		page_pool_recycle_direct(pool, page);

	}



	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len,

			 DMA_FROM_DEVICE);

	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);



	dev_kfree_skb_any(skb);

}



static int prueth_tx_ts_cookie_get(struct prueth_emac *emac)
@@ -662,13 +859,13 @@ enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev 
	struct prueth_emac *emac = netdev_priv(ndev);

	struct prueth *prueth = emac->prueth;

	struct netdev_queue *netif_txq;

	struct prueth_swdata *swdata;

	struct prueth_tx_chn *tx_chn;

	dma_addr_t desc_dma, buf_dma;

	u32 pkt_len, dst_tag_id;

	int i, ret = 0, q_idx;

	bool in_tx_ts = 0;

	int tx_ts_cookie;

	void **swdata;

	u32 *epib;



	pkt_len = skb_headlen(skb);
@@ -730,7 +927,8 @@ enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev 
	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);

	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);

	swdata = cppi5_hdesc_get_swdata(first_desc);

	*swdata = skb;

	swdata->type = PRUETH_SWDATA_SKB;

	swdata->data.skb = skb;



	/* Handle the case where skb is fragmented in pages */

	cur_desc = first_desc;
@@ -833,15 +1031,27 @@ static void prueth_tx_cleanup(void *data, dma_addr_t desc_dma) 
{

	struct prueth_tx_chn *tx_chn = data;

	struct cppi5_host_desc_t *desc_tx;

	struct prueth_swdata *swdata;

	struct xdp_frame *xdpf;

	struct sk_buff *skb;

	void **swdata;



	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);

	swdata = cppi5_hdesc_get_swdata(desc_tx);

	skb = *(swdata);

	prueth_xmit_free(tx_chn, desc_tx);



	switch (swdata->type) {

	case PRUETH_SWDATA_SKB:

		skb = swdata->data.skb;

		dev_kfree_skb_any(skb);

		break;

	case PRUETH_SWDATA_XDPF:

		xdpf = swdata->data.xdpf;

		xdp_return_frame(xdpf);

		break;

	default:

		break;

	}



	prueth_xmit_free(tx_chn, desc_tx);

}



irqreturn_t prueth_rx_irq(int irq, void *dev_id)
@@ -875,15 +1085,18 @@ int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget) 
		PRUETH_RX_FLOW_DATA_SR1 : PRUETH_RX_FLOW_DATA;

	int flow = emac->is_sr1 ?

		PRUETH_MAX_RX_FLOWS_SR1 : PRUETH_MAX_RX_FLOWS;

	int xdp_state_or = 0;

	int num_rx = 0;

	int cur_budget;

	u32 xdp_state;

	int ret;



	while (flow--) {

		cur_budget = budget - num_rx;



		while (cur_budget--) {

			ret = emac_rx_packet(emac, flow);

			ret = emac_rx_packet(emac, flow, &xdp_state);

			xdp_state_or |= xdp_state;

			if (ret)

				break;

			num_rx++;
@@ -893,6 +1106,9 @@ int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget) 
			break;

	}



	if (xdp_state_or & ICSSG_XDP_REDIR)

		xdp_do_flush();



	if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {

		if (unlikely(emac->rx_pace_timeout_ns)) {

			hrtimer_start(&emac->rx_hrtimer,
@@ -907,29 +1123,71 @@ int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget) 
}

EXPORT_SYMBOL_GPL(icssg_napi_rx_poll);



static struct page_pool *prueth_create_page_pool(struct prueth_emac *emac,

						 struct device *dma_dev,

						 int size)

{

	struct page_pool_params pp_params = { 0 };

	struct page_pool *pool;



	pp_params.order = 0;

	pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;

	pp_params.pool_size = size;

	pp_params.nid = dev_to_node(emac->prueth->dev);

	pp_params.dma_dir = DMA_BIDIRECTIONAL;

	pp_params.dev = dma_dev;

	pp_params.napi = &emac->napi_rx;

	pp_params.max_len = PAGE_SIZE;



	pool = page_pool_create(&pp_params);

	if (IS_ERR(pool))

		netdev_err(emac->ndev, "cannot create rx page pool\n");



	return pool;

}



int prueth_prepare_rx_chan(struct prueth_emac *emac,

			   struct prueth_rx_chn *chn,

			   int buf_size)

{

	struct sk_buff *skb;

	struct page_pool *pool;

	struct page *page;

	int i, ret;



	pool = prueth_create_page_pool(emac, chn->dma_dev, chn->descs_num);

	if (IS_ERR(pool))

		return PTR_ERR(pool);



	chn->pg_pool = pool;



	for (i = 0; i < chn->descs_num; i++) {

		skb = __netdev_alloc_skb_ip_align(NULL, buf_size, GFP_KERNEL);

		if (!skb)

			return -ENOMEM;

		/* NOTE: we're not using memory efficiently here.

		 * 1 full page (4KB?) used here instead of

		 * PRUETH_MAX_PKT_SIZE (~1.5KB?)

		 */

		page = page_pool_dev_alloc_pages(pool);

		if (!page) {

			netdev_err(emac->ndev, "couldn't allocate rx page\n");

			ret = -ENOMEM;

			goto recycle_alloc_pg;

		}



		ret = prueth_dma_rx_push(emac, skb, chn);

		ret = prueth_dma_rx_push_mapped(emac, chn, page, buf_size);

		if (ret < 0) {

			netdev_err(emac->ndev,

				   "cannot submit skb for rx chan %s ret %d\n",

				   "cannot submit page for rx chan %s ret %d\n",

				   chn->name, ret);

			kfree_skb(skb);

			return ret;

			page_pool_recycle_direct(pool, page);

			goto recycle_alloc_pg;

		}

	}



	return 0;



recycle_alloc_pg:

	prueth_reset_rx_chan(&emac->rx_chns, PRUETH_MAX_RX_FLOWS, false);



	return ret;

}

EXPORT_SYMBOL_GPL(prueth_prepare_rx_chan);
@@ -958,6 +1216,9 @@ void prueth_reset_rx_chan(struct prueth_rx_chn *chn, 
					  prueth_rx_cleanup, !!i);

	if (disable)

		k3_udma_glue_disable_rx_chn(chn->rx_chn);



	page_pool_destroy(chn->pg_pool);

	chn->pg_pool = NULL;

}

EXPORT_SYMBOL_GPL(prueth_reset_rx_chan);

drivers/net/ethernet/ti/icssg/icssg_prueth.c

+129 −2
Original line number Diff line number Diff line
@@ -559,6 +559,33 @@ const struct icss_iep_clockops prueth_iep_clockops = { 
	.perout_enable = prueth_perout_enable,

};



static int prueth_create_xdp_rxqs(struct prueth_emac *emac)

{

	struct xdp_rxq_info *rxq = &emac->rx_chns.xdp_rxq;

	struct page_pool *pool = emac->rx_chns.pg_pool;

	int ret;



	ret = xdp_rxq_info_reg(rxq, emac->ndev, 0, emac->napi_rx.napi_id);

	if (ret)

		return ret;



	ret = xdp_rxq_info_reg_mem_model(rxq, MEM_TYPE_PAGE_POOL, pool);

	if (ret)

		xdp_rxq_info_unreg(rxq);



	return ret;

}



static void prueth_destroy_xdp_rxqs(struct prueth_emac *emac)

{

	struct xdp_rxq_info *rxq = &emac->rx_chns.xdp_rxq;



	if (!xdp_rxq_info_is_reg(rxq))

		return;



	xdp_rxq_info_unreg(rxq);

}



static int icssg_prueth_add_mcast(struct net_device *ndev, const u8 *addr)

{

	struct net_device *real_dev;
@@ -780,10 +807,14 @@ static int emac_ndo_open(struct net_device *ndev) 
	if (ret)

		goto free_tx_ts_irq;



	ret = k3_udma_glue_enable_rx_chn(emac->rx_chns.rx_chn);

	ret = prueth_create_xdp_rxqs(emac);

	if (ret)

		goto reset_rx_chn;



	ret = k3_udma_glue_enable_rx_chn(emac->rx_chns.rx_chn);

	if (ret)

		goto destroy_xdp_rxqs;



	for (i = 0; i < emac->tx_ch_num; i++) {

		ret = k3_udma_glue_enable_tx_chn(emac->tx_chns[i].tx_chn);

		if (ret)
@@ -809,6 +840,8 @@ static int emac_ndo_open(struct net_device *ndev) 
	 * any SKB for completion. So set false to free_skb

	 */

	prueth_reset_tx_chan(emac, i, false);

destroy_xdp_rxqs:

	prueth_destroy_xdp_rxqs(emac);

reset_rx_chn:

	prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, false);

free_tx_ts_irq:
@@ -879,7 +912,7 @@ static int emac_ndo_stop(struct net_device *ndev) 
	k3_udma_glue_tdown_rx_chn(emac->rx_chns.rx_chn, true);



	prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, true);



	prueth_destroy_xdp_rxqs(emac);

	napi_disable(&emac->napi_rx);

	hrtimer_cancel(&emac->rx_hrtimer);
@@ -1024,6 +1057,93 @@ static int emac_ndo_vlan_rx_del_vid(struct net_device *ndev, 
	return 0;

}



/**

 * emac_xdp_xmit - Implements ndo_xdp_xmit

 * @dev: netdev

 * @n: number of frames

 * @frames: array of XDP buffer pointers

 * @flags: XDP extra info

 *

 * Return: number of frames successfully sent. Failed frames

 * will be free'ed by XDP core.

 *

 * For error cases, a negative errno code is returned and no-frames

 * are transmitted (caller must handle freeing frames).

 **/

static int emac_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,

			 u32 flags)

{

	struct prueth_emac *emac = netdev_priv(dev);

	struct net_device *ndev = emac->ndev;

	struct xdp_frame *xdpf;

	unsigned int q_idx;

	int nxmit = 0;

	u32 err;

	int i;



	q_idx = smp_processor_id() % emac->tx_ch_num;



	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))

		return -EINVAL;



	for (i = 0; i < n; i++) {

		xdpf = frames[i];

		err = emac_xmit_xdp_frame(emac, xdpf, NULL, q_idx);

		if (err != ICSSG_XDP_TX) {

			ndev->stats.tx_dropped++;

			break;

		}

		nxmit++;

	}



	return nxmit;

}



/**

 * emac_xdp_setup - add/remove an XDP program

 * @emac: emac device

 * @bpf: XDP program

 *

 * Return: Always 0 (Success)

 **/

static int emac_xdp_setup(struct prueth_emac *emac, struct netdev_bpf *bpf)

{

	struct bpf_prog *prog = bpf->prog;

	xdp_features_t val;



	val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |

	      NETDEV_XDP_ACT_NDO_XMIT;

	xdp_set_features_flag(emac->ndev, val);



	if (!emac->xdpi.prog && !prog)

		return 0;



	WRITE_ONCE(emac->xdp_prog, prog);



	xdp_attachment_setup(&emac->xdpi, bpf);



	return 0;

}



/**

 * emac_ndo_bpf - implements ndo_bpf for icssg_prueth

 * @ndev: network adapter device

 * @bpf: XDP program

 *

 * Return: 0 on success, error code on failure.

 **/

static int emac_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf)

{

	struct prueth_emac *emac = netdev_priv(ndev);



	switch (bpf->command) {

	case XDP_SETUP_PROG:

		return emac_xdp_setup(emac, bpf);

	default:

		return -EINVAL;

	}

}



static const struct net_device_ops emac_netdev_ops = {

	.ndo_open = emac_ndo_open,

	.ndo_stop = emac_ndo_stop,
@@ -1038,6 +1158,8 @@ static const struct net_device_ops emac_netdev_ops = { 
	.ndo_fix_features = emac_ndo_fix_features,

	.ndo_vlan_rx_add_vid = emac_ndo_vlan_rx_add_vid,

	.ndo_vlan_rx_kill_vid = emac_ndo_vlan_rx_del_vid,

	.ndo_bpf = emac_ndo_bpf,

	.ndo_xdp_xmit = emac_xdp_xmit,

};



static int prueth_netdev_init(struct prueth *prueth,
@@ -1066,6 +1188,8 @@ static int prueth_netdev_init(struct prueth *prueth, 
	emac->prueth = prueth;

	emac->ndev = ndev;

	emac->port_id = port;

	emac->xdp_prog = NULL;

	emac->ndev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

	emac->cmd_wq = create_singlethread_workqueue("icssg_cmd_wq");

	if (!emac->cmd_wq) {

		ret = -ENOMEM;
@@ -1522,6 +1646,9 @@ static int prueth_probe(struct platform_device *pdev) 


	np = dev->of_node;



	BUILD_BUG_ON_MSG((sizeof(struct prueth_swdata) > PRUETH_NAV_SW_DATA_SIZE),

			 "insufficient SW_DATA size");



	prueth = devm_kzalloc(dev, sizeof(*prueth), GFP_KERNEL);

	if (!prueth)

		return -ENOMEM;

drivers/net/ethernet/ti/icssg/icssg_prueth.h

+44 −3
Original line number Diff line number Diff line
@@ -8,6 +8,8 @@ 
#ifndef __NET_TI_ICSSG_PRUETH_H

#define __NET_TI_ICSSG_PRUETH_H



#include <linux/bpf.h>

#include <linux/bpf_trace.h>

#include <linux/etherdevice.h>

#include <linux/genalloc.h>

#include <linux/if_vlan.h>
@@ -33,6 +35,8 @@ 
#include <linux/dma/k3-udma-glue.h>



#include <net/devlink.h>

#include <net/xdp.h>

#include <net/page_pool/helpers.h>



#include "icssg_config.h"

#include "icss_iep.h"
@@ -131,6 +135,26 @@ struct prueth_rx_chn { 
	u32 descs_num;

	unsigned int irq[ICSSG_MAX_RFLOWS];	/* separate irq per flow */

	char name[32];

	struct page_pool *pg_pool;

	struct xdp_rxq_info xdp_rxq;

};



enum prueth_swdata_type {

	PRUETH_SWDATA_INVALID = 0,

	PRUETH_SWDATA_SKB,

	PRUETH_SWDATA_PAGE,

	PRUETH_SWDATA_CMD,

	PRUETH_SWDATA_XDPF,

};



struct prueth_swdata {

	enum prueth_swdata_type type;

	union prueth_data {

		struct sk_buff *skb;

		struct page *page;

		u32 cmd;

		struct xdp_frame *xdpf;

	} data;

};



/* There are 4 Tx DMA channels, but the highest priority is CH3 (thread 3)
@@ -140,6 +164,12 @@ struct prueth_rx_chn { 


#define PRUETH_MAX_TX_TS_REQUESTS	50 /* Max simultaneous TX_TS requests */



/* XDP BPF state */

#define ICSSG_XDP_PASS           0

#define ICSSG_XDP_CONSUMED       BIT(0)

#define ICSSG_XDP_TX             BIT(1)

#define ICSSG_XDP_REDIR          BIT(2)



/* Minimum coalesce time in usecs for both Tx and Rx */

#define ICSSG_MIN_COALESCE_USECS 20
@@ -208,8 +238,14 @@ struct prueth_emac { 
	unsigned long rx_pace_timeout_ns;



	struct netdev_hw_addr_list vlan_mcast_list[MAX_VLAN_ID];

	struct bpf_prog *xdp_prog;

	struct xdp_attachment_info xdpi;

};



/* The buf includes headroom compatible with both skb and xdpf */

#define PRUETH_HEADROOM_NA (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + NET_IP_ALIGN)

#define PRUETH_HEADROOM  ALIGN(PRUETH_HEADROOM_NA, sizeof(long))



/**

 * struct prueth_pdata - PRUeth platform data

 * @fdqring_mode: Free desc queue mode
@@ -410,9 +446,10 @@ int prueth_init_rx_chns(struct prueth_emac *emac, 
			struct prueth_rx_chn *rx_chn,

			char *name, u32 max_rflows,

			u32 max_desc_num);

int prueth_dma_rx_push(struct prueth_emac *emac,

		       struct sk_buff *skb,

		       struct prueth_rx_chn *rx_chn);

int prueth_dma_rx_push_mapped(struct prueth_emac *emac,

			      struct prueth_rx_chn *rx_chn,

			      struct page *page, u32 buf_len);

unsigned int prueth_rxbuf_total_len(unsigned int len);

void emac_rx_timestamp(struct prueth_emac *emac,

		       struct sk_buff *skb, u32 *psdata);

enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev);
@@ -441,5 +478,9 @@ void prueth_put_cores(struct prueth *prueth, int slice); 


/* Revision specific helper */

u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns);

u32 emac_xmit_xdp_frame(struct prueth_emac *emac,

			struct xdp_frame *xdpf,

			struct page *page,

			unsigned int q_idx);



#endif /* __NET_TI_ICSSG_PRUETH_H */

drivers/net/ethernet/ti/icssg/icssg_prueth_sr1.c

+28 −30

File changed.

Preview size limit exceeded, changes collapsed.