Commit 8eeed837 authored by Shannon Nelson's avatar Shannon Nelson Committed by David S. Miller
Browse files

ionic: Add XDP_TX support 



The XDP_TX packets get fed back into the Rx queue's partnered
Tx queue as an xdp_frame.

Co-developed-by: default avatarBrett Creeley <brett.creeley@amd.com>
Signed-off-by: default avatarBrett Creeley <brett.creeley@amd.com>
Signed-off-by: default avatarShannon Nelson <shannon.nelson@amd.com>
Reviewed-by: default avatarJacob Keller <jacob.e.keller@intel.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent f81da39b

drivers/net/ethernet/pensando/ionic/ionic_dev.h

+3 −0
Original line number Diff line number Diff line
@@ -228,6 +228,8 @@ struct ionic_desc_info { 
	struct ionic_buf_info bufs[MAX_SKB_FRAGS + 1];

	ionic_desc_cb cb;

	void *cb_arg;

	struct xdp_frame *xdpf;

	enum xdp_action act;

};



#define IONIC_QUEUE_NAME_MAX_SZ		16
@@ -263,6 +265,7 @@ struct ionic_queue { 
		struct ionic_rxq_sg_desc *rxq_sgl;

	};

	struct xdp_rxq_info *xdp_rxq_info;

	struct ionic_queue *partner;

	dma_addr_t base_pa;

	dma_addr_t cmb_base_pa;

	dma_addr_t sg_base_pa;

drivers/net/ethernet/pensando/ionic/ionic_lif.c

+4 −0
Original line number Diff line number Diff line
@@ -878,6 +878,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) 
	};

	int err;



	q->partner = &lif->txqcqs[q->index]->q;

	q->partner->partner = q;



	if (!lif->xdp_prog)

		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
@@ -2926,6 +2929,7 @@ static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b) 
	swap(a->q.base_pa,    b->q.base_pa);

	swap(a->q.info,       b->q.info);

	swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);

	swap(a->q.partner,    b->q.partner);

	swap(a->q_base,       b->q_base);

	swap(a->q_base_pa,    b->q_base_pa);

	swap(a->q_size,       b->q_size);

drivers/net/ethernet/pensando/ionic/ionic_lif.h

+4 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ struct ionic_tx_stats { 
	u64 dma_map_err;

	u64 hwstamp_valid;

	u64 hwstamp_invalid;

	u64 xdp_frames;

};



struct ionic_rx_stats {
@@ -54,6 +55,7 @@ struct ionic_rx_stats { 
	u64 xdp_drop;

	u64 xdp_aborted;

	u64 xdp_pass;

	u64 xdp_tx;

};



#define IONIC_QCQ_F_INITED		BIT(0)
@@ -141,6 +143,8 @@ struct ionic_lif_sw_stats { 
	u64 xdp_drop;

	u64 xdp_aborted;

	u64 xdp_pass;

	u64 xdp_tx;

	u64 xdp_frames;

};



enum ionic_lif_state_flags {

drivers/net/ethernet/pensando/ionic/ionic_stats.c

+6 −0
Original line number Diff line number Diff line
@@ -30,6 +30,8 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = { 
	IONIC_LIF_STAT_DESC(xdp_drop),

	IONIC_LIF_STAT_DESC(xdp_aborted),

	IONIC_LIF_STAT_DESC(xdp_pass),

	IONIC_LIF_STAT_DESC(xdp_tx),

	IONIC_LIF_STAT_DESC(xdp_frames),

};



static const struct ionic_stat_desc ionic_port_stats_desc[] = {
@@ -138,6 +140,7 @@ static const struct ionic_stat_desc ionic_tx_stats_desc[] = { 
	IONIC_TX_STAT_DESC(csum_none),

	IONIC_TX_STAT_DESC(csum),

	IONIC_TX_STAT_DESC(vlan_inserted),

	IONIC_TX_STAT_DESC(xdp_frames),

};



static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
@@ -155,6 +158,7 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = { 
	IONIC_RX_STAT_DESC(xdp_drop),

	IONIC_RX_STAT_DESC(xdp_aborted),

	IONIC_RX_STAT_DESC(xdp_pass),

	IONIC_RX_STAT_DESC(xdp_tx),

};



#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -177,6 +181,7 @@ static void ionic_add_lif_txq_stats(struct ionic_lif *lif, int q_num, 
	stats->tx_csum += txstats->csum;

	stats->tx_hwstamp_valid += txstats->hwstamp_valid;

	stats->tx_hwstamp_invalid += txstats->hwstamp_invalid;

	stats->xdp_frames += txstats->xdp_frames;

}



static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
@@ -194,6 +199,7 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num, 
	stats->xdp_drop += rxstats->xdp_drop;

	stats->xdp_aborted += rxstats->xdp_aborted;

	stats->xdp_pass += rxstats->xdp_pass;

	stats->xdp_tx += rxstats->xdp_tx;

}



static void ionic_get_lif_stats(struct ionic_lif *lif,

drivers/net/ethernet/pensando/ionic/ionic_txrx.c

+133 −3
Original line number Diff line number Diff line
@@ -10,6 +10,16 @@ 
#include "ionic_lif.h"

#include "ionic_txrx.h"



static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs);



static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,

				      void *data, size_t len);



static void ionic_tx_clean(struct ionic_queue *q,

			   struct ionic_desc_info *desc_info,

			   struct ionic_cq_info *cq_info,

			   void *cb_arg);



static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,

				  ionic_desc_cb cb_func, void *cb_arg)

{
@@ -297,6 +307,75 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 
	return skb;

}



static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,

				    struct ionic_desc_info *desc_info)

{

	unsigned int nbufs = desc_info->nbufs;

	struct ionic_buf_info *buf_info;

	struct device *dev = q->dev;



	if (!nbufs)

		return;



	buf_info = desc_info->bufs;

	dma_unmap_single(dev, buf_info->dma_addr,

			 buf_info->len, DMA_TO_DEVICE);

	__free_pages(buf_info->page, 0);

	buf_info->page = NULL;



	desc_info->nbufs = 0;

	desc_info->xdpf = NULL;

	desc_info->act = 0;

}



static int ionic_xdp_post_frame(struct net_device *netdev,

				struct ionic_queue *q, struct xdp_frame *frame,

				enum xdp_action act, struct page *page, int off,

				bool ring_doorbell)

{

	struct ionic_desc_info *desc_info;

	struct ionic_buf_info *buf_info;

	struct ionic_tx_stats *stats;

	struct ionic_txq_desc *desc;

	size_t len = frame->len;

	dma_addr_t dma_addr;

	u64 cmd;



	desc_info = &q->info[q->head_idx];

	desc = desc_info->txq_desc;

	buf_info = desc_info->bufs;

	stats = q_to_tx_stats(q);



	dma_addr = ionic_tx_map_single(q, frame->data, len);

	if (dma_mapping_error(q->dev, dma_addr)) {

		stats->dma_map_err++;

		return -EIO;

	}

	buf_info->dma_addr = dma_addr;

	buf_info->len = len;

	buf_info->page = page;

	buf_info->page_offset = off;



	desc_info->nbufs = 1;

	desc_info->xdpf = frame;

	desc_info->act = act;



	cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,

				  0, 0, buf_info->dma_addr);

	desc->cmd = cpu_to_le64(cmd);

	desc->len = cpu_to_le16(len);

	desc->csum_start = 0;

	desc->csum_offset = 0;



	stats->xdp_frames++;

	stats->pkts++;

	stats->bytes += len;



	ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL);



	return 0;

}



static bool ionic_run_xdp(struct ionic_rx_stats *stats,

			  struct net_device *netdev,

			  struct bpf_prog *xdp_prog,
@@ -306,6 +385,10 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, 
{

	u32 xdp_action = XDP_ABORTED;

	struct xdp_buff xdp_buf;

	struct ionic_queue *txq;

	struct netdev_queue *nq;

	struct xdp_frame *xdpf;

	int err = 0;



	xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);

	xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
@@ -330,13 +413,50 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, 
		break;



	case XDP_TX:

		xdpf = xdp_convert_buff_to_frame(&xdp_buf);

		if (!xdpf)

			goto out_xdp_abort;



		txq = rxq->partner;

		nq = netdev_get_tx_queue(netdev, txq->index);

		__netif_tx_lock(nq, smp_processor_id());

		txq_trans_cond_update(nq);



		if (netif_tx_queue_stopped(nq) ||

		    unlikely(ionic_maybe_stop_tx(txq, 1))) {

			__netif_tx_unlock(nq);

			goto out_xdp_abort;

		}



		dma_unmap_page(rxq->dev, buf_info->dma_addr,

			       IONIC_PAGE_SIZE, DMA_FROM_DEVICE);



		err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX,

					   buf_info->page,

					   buf_info->page_offset,

					   true);

		__netif_tx_unlock(nq);

		if (err) {

			netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);

			goto out_xdp_abort;

		}

		stats->xdp_tx++;



		/* the Tx completion will free the buffers */

		break;



	case XDP_REDIRECT:

	case XDP_ABORTED:

	default:

		goto out_xdp_abort;

	}



	return true;



out_xdp_abort:

	trace_xdp_exception(netdev, xdp_prog, xdp_action);

	ionic_rx_page_free(rxq, buf_info);

	stats->xdp_aborted++;

	}



	return true;

}
@@ -893,6 +1013,16 @@ static void ionic_tx_clean(struct ionic_queue *q, 
	struct sk_buff *skb = cb_arg;

	u16 qi;



	if (desc_info->xdpf) {

		ionic_xdp_tx_desc_clean(q->partner, desc_info);

		stats->clean++;



		if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))

			netif_wake_subqueue(q->lif->netdev, q->index);



		return;

	}



	ionic_tx_desc_unmap_bufs(q, desc_info);



	if (!skb)