Commit 53616af0 authored by Suman Ghosh's avatar Suman Ghosh Committed by Paolo Abeni
Browse files

octeontx2-pf: AF_XDP zero copy transmit support 



This patch implements below changes,

1. To avoid concurrency with normal traffic uses
   XDP queues.
2. Since there are chances that XDP and AF_XDP can
   fall under same queue uses separate flags to handle
   dma buffers.

Signed-off-by: default avatarHariprasad Kelam <hkelam@marvell.com>
Signed-off-by: default avatarSuman Ghosh <sumang@marvell.com>
Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent c5c2398e

drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c

+4 −0
Original line number Diff line number Diff line
@@ -1037,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) 


	sq->stats.bytes = 0;

	sq->stats.pkts = 0;

	/* Attach XSK_BUFF_POOL to XDP queue */

	if (qidx > pfvf->hw.xdp_queues)

		otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));





	chan_offset = qidx % pfvf->hw.tx_chan_cnt;

	err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);

drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h

+6 −0
Original line number Diff line number Diff line
@@ -129,6 +129,12 @@ enum otx2_errcodes_re { 
	ERRCODE_IL4_CSUM = 0x22,

};



enum otx2_xdp_action {

	OTX2_XDP_TX	  = BIT(0),

	OTX2_XDP_REDIRECT = BIT(1),

	OTX2_AF_XDP_FRAME = BIT(2),

};



struct otx2_dev_stats {

	u64 rx_bytes;

	u64 rx_frames;

drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c

+1 −1
Original line number Diff line number Diff line
@@ -2693,7 +2693,7 @@ static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf, 
		return -ENOMEM;



	err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,

				     qidx, XDP_REDIRECT);

				     qidx, OTX2_XDP_REDIRECT);

	if (!err) {

		otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);

		xdp_return_frame(xdpf);

drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c

+39 −10
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include "otx2_txrx.h"

#include "otx2_ptp.h"

#include "cn10k.h"

#include "otx2_xsk.h"



#define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))

#define PTP_PORT	        0x13F
@@ -103,13 +104,19 @@ static unsigned int frag_num(unsigned int i) 


static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,

				     struct otx2_snd_queue *sq,

				     struct nix_cqe_tx_s *cqe)

				     struct nix_cqe_tx_s *cqe,

				     int *xsk_frames)

{

	struct nix_send_comp_s *snd_comp = &cqe->comp;

	struct sg_list *sg;



	sg = &sq->sg[snd_comp->sqe_id];

	if (sg->flags & XDP_REDIRECT)

	if (sg->flags & OTX2_AF_XDP_FRAME) {

		(*xsk_frames)++;

		return;

	}



	if (sg->flags & OTX2_XDP_REDIRECT)

		otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);

	xdp_return_frame((struct xdp_frame *)sg->skb);

	sg->skb = (u64)NULL;
@@ -434,6 +441,18 @@ int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 
	return cnt - cq->pool_ptrs;

}



static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,

				int *xsk_frames, int qidx, int budget)

{

	if (*xsk_frames)

		xsk_tx_completed(xsk_pool, *xsk_frames);



	if (xsk_uses_need_wakeup(xsk_pool))

		xsk_set_tx_need_wakeup(xsk_pool);



	otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);

}



static int otx2_tx_napi_handler(struct otx2_nic *pfvf,

				struct otx2_cq_queue *cq, int budget)

{
@@ -442,16 +461,22 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, 
	struct nix_cqe_tx_s *cqe;

	struct net_device *ndev;

	int processed_cqe = 0;

	int xsk_frames = 0;



	qidx = cq->cq_idx - pfvf->hw.rx_queues;

	sq = &pfvf->qset.sq[qidx];



	if (cq->pend_cqe >= budget)

		goto process_cqe;



	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)

	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {

		if (sq->xsk_pool)

			otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,

					    qidx, budget);

		return 0;

	}



process_cqe:

	qidx = cq->cq_idx - pfvf->hw.rx_queues;

	sq = &pfvf->qset.sq[qidx];



	while (likely(processed_cqe < budget) && cq->pend_cqe) {

		cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
@@ -461,10 +486,8 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, 
			break;

		}



		qidx = cq->cq_idx - pfvf->hw.rx_queues;



		if (cq->cq_type == CQ_XDP)

			otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);

			otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);

		else

			otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],

					     cqe, budget, &tx_pkts, &tx_bytes);
@@ -505,6 +528,10 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, 
		    netif_carrier_ok(ndev))

			netif_tx_wake_queue(txq);

	}



	if (sq->xsk_pool)

		otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);



	return 0;

}
@@ -1499,8 +1526,10 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, 
		qidx += pfvf->hw.tx_queues;

		cq->pool_ptrs++;

		xdpf = xdp_convert_buff_to_frame(&xdp);

		return otx2_xdp_sq_append_pkt(pfvf, xdpf, cqe->sg.seg_addr,

					      cqe->sg.seg_size, qidx, XDP_TX);

		return otx2_xdp_sq_append_pkt(pfvf, xdpf,

					      cqe->sg.seg_addr,

					      cqe->sg.seg_size,

					      qidx, OTX2_XDP_TX);

	case XDP_REDIRECT:

		cq->pool_ptrs++;

		if (xsk_buff) {

drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h

+2 −0
Original line number Diff line number Diff line
@@ -106,6 +106,8 @@ struct otx2_snd_queue { 
	/* SQE ring and CPT response queue for Inline IPSEC */

	struct qmem		*sqe_ring;

	struct qmem		*cpt_resp;

	/* Buffer pool for af_xdp zero-copy */

	struct xsk_buff_pool    *xsk_pool;

} ____cacheline_aligned_in_smp;



enum cq_type {