Commit b4cb7308 authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'add-af_xdp-support-for-cn10k' 

Suman Ghosh says:

====================
Add af_xdp support for cn10k

This patchset includes changes to support AF_XDP for cn10k chipsets. Both
non-zero copy and zero copy will be supported after these changes. Also,
the RSS will be reconfigured once a particular receive queue is
added/removed to/from AF_XDP support.

Patch #1: octeontx2-pf: use xdp_return_frame() to free xdp buffers

Patch #2: octeontx2-pf: Add AF_XDP non-zero copy support

Patch #3: octeontx2-pf: AF_XDP zero copy receive support

Patch #4: octeontx2-pf: Reconfigure RSS table after enabling AF_XDP
zerocopy on rx queue

Patch #5: octeontx2-pf: Prepare for AF_XDP transmit

Patch #6: octeontx2-pf: AF_XDP zero copy transmit support
====================

Link: https://patch.msgid.link/20250213053141.2833254-1-sumang@marvell.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 0f375d90 53616af0

drivers/net/ethernet/marvell/octeontx2/nic/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ obj-$(CONFIG_RVU_ESWITCH) += rvu_rep.o 


rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \

               otx2_flows.o otx2_tc.o cn10k.o otx2_dmac_flt.o \

               otx2_devlink.o qos_sq.o qos.o

               otx2_devlink.o qos_sq.o qos.o otx2_xsk.o

rvu_nicvf-y := otx2_vf.o

rvu_rep-y := rep.o

drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c

+6 −1
Original line number Diff line number Diff line
@@ -112,9 +112,12 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 
	struct otx2_nic *pfvf = dev;

	int cnt = cq->pool_ptrs;

	u64 ptrs[NPA_MAX_BURST];

	struct otx2_pool *pool;

	dma_addr_t bufptr;

	int num_ptrs = 1;



	pool = &pfvf->qset.pool[cq->cq_idx];



	/* Refill pool with new buffers */

	while (cq->pool_ptrs) {

		if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
@@ -124,7 +127,9 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 
			break;

		}

		cq->pool_ptrs--;

		ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;

		ptrs[num_ptrs] = pool->xsk_pool ?

				 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;



		num_ptrs++;

		if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {

			__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,

drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c

+94 −28
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include "otx2_common.h"

#include "otx2_struct.h"

#include "cn10k.h"

#include "otx2_xsk.h"



static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf)

{
@@ -330,6 +331,10 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id) 
	rss_ctx = rss->rss_ctx[ctx_id];

	/* Get memory to put this msg */

	for (idx = 0; idx < rss->rss_size; idx++) {

		/* Ignore the queue if AF_XDP zero copy is enabled */

		if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))

			continue;



		aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox);

		if (!aq) {

			/* The shared memory buffer can be full.
@@ -549,10 +554,13 @@ static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, 
}



static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,

			     dma_addr_t *dma)

			     dma_addr_t *dma, int qidx, int idx)

{

	u8 *buf;



	if (pool->xsk_pool)

		return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx);



	if (pool->page_pool)

		return otx2_alloc_pool_buf(pfvf, pool, dma);
@@ -571,12 +579,12 @@ static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, 
}



int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,

		    dma_addr_t *dma)

		    dma_addr_t *dma, int qidx, int idx)

{

	int ret;



	local_bh_disable();

	ret = __otx2_alloc_rbuf(pfvf, pool, dma);

	ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx);

	local_bh_enable();

	return ret;

}
@@ -584,7 +592,8 @@ int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, 
int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq,

		      dma_addr_t *dma)

{

	if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma)))

	if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma,

				       cq->cq_idx, cq->pool_ptrs - 1)))

		return -ENOMEM;

	return 0;

}
@@ -884,7 +893,7 @@ void otx2_sqb_flush(struct otx2_nic *pfvf) 
#define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */

#define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */



static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)

int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)

{

	struct otx2_qset *qset = &pfvf->qset;

	struct nix_aq_enq_req *aq;
@@ -1028,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) 


	sq->stats.bytes = 0;

	sq->stats.pkts = 0;

	/* Attach XSK_BUFF_POOL to XDP queue */

	if (qidx > pfvf->hw.xdp_queues)

		otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));





	chan_offset = qidx % pfvf->hw.tx_chan_cnt;

	err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);
@@ -1041,12 +1054,13 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) 


}



static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)

int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)

{

	struct otx2_qset *qset = &pfvf->qset;

	int err, pool_id, non_xdp_queues;

	struct nix_aq_enq_req *aq;

	struct otx2_cq_queue *cq;

	struct otx2_pool *pool;



	cq = &qset->cq[qidx];

	cq->cq_idx = qidx;
@@ -1055,8 +1069,20 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) 
		cq->cq_type = CQ_RX;

		cq->cint_idx = qidx;

		cq->cqe_cnt = qset->rqe_cnt;

		if (pfvf->xdp_prog)

		if (pfvf->xdp_prog) {

			xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0);

			pool = &qset->pool[qidx];

			if (pool->xsk_pool) {

				xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,

							   MEM_TYPE_XSK_BUFF_POOL,

							   NULL);

				xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq);

			} else if (pool->page_pool) {

				xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,

							   MEM_TYPE_PAGE_POOL,

							   pool->page_pool);

			}

		}

	} else if (qidx < non_xdp_queues) {

		cq->cq_type = CQ_TX;

		cq->cint_idx = qidx - pfvf->hw.rx_queues;
@@ -1275,9 +1301,10 @@ void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool, 


	pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);

	page = virt_to_head_page(phys_to_virt(pa));



	if (pool->page_pool) {

		page_pool_put_full_page(pool->page_pool, page, true);

	} else if (pool->xsk_pool) {

		/* Note: No way of identifying xdp_buff */

	} else {

		dma_unmap_page_attrs(pfvf->dev, iova, size,

				     DMA_FROM_DEVICE,
@@ -1292,6 +1319,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) 
	int pool_id, pool_start = 0, pool_end = 0, size = 0;

	struct otx2_pool *pool;

	u64 iova;

	int idx;



	if (type == AURA_NIX_SQ) {

		pool_start = otx2_get_pool_idx(pfvf, type, 0);
@@ -1306,16 +1334,21 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) 


	/* Free SQB and RQB pointers from the aura pool */

	for (pool_id = pool_start; pool_id < pool_end; pool_id++) {

		iova = otx2_aura_allocptr(pfvf, pool_id);

		pool = &pfvf->qset.pool[pool_id];

		iova = otx2_aura_allocptr(pfvf, pool_id);

		while (iova) {

			if (type == AURA_NIX_RQ)

				iova -= OTX2_HEAD_ROOM;



			otx2_free_bufs(pfvf, pool, iova, size);



			iova = otx2_aura_allocptr(pfvf, pool_id);

		}



		for (idx = 0 ; idx < pool->xdp_cnt; idx++) {

			if (!pool->xdp[idx])

				continue;



			xsk_buff_free(pool->xdp[idx]);

		}

	}

}
@@ -1332,7 +1365,8 @@ void otx2_aura_pool_free(struct otx2_nic *pfvf) 
		qmem_free(pfvf->dev, pool->stack);

		qmem_free(pfvf->dev, pool->fc_addr);

		page_pool_destroy(pool->page_pool);

		pool->page_pool = NULL;

		devm_kfree(pfvf->dev, pool->xdp);

		pool->xsk_pool = NULL;

	}

	devm_kfree(pfvf->dev, pfvf->qset.pool);

	pfvf->qset.pool = NULL;
@@ -1419,6 +1453,7 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, 
		   int stack_pages, int numptrs, int buf_size, int type)

{

	struct page_pool_params pp_params = { 0 };

	struct xsk_buff_pool *xsk_pool;

	struct npa_aq_enq_req *aq;

	struct otx2_pool *pool;

	int err;
@@ -1462,11 +1497,10 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, 
	aq->ctype = NPA_AQ_CTYPE_POOL;

	aq->op = NPA_AQ_INSTOP_INIT;



	if (type != AURA_NIX_RQ) {

		pool->page_pool = NULL;

	if (type != AURA_NIX_RQ)

		return 0;

	}



	if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) {

		pp_params.order = get_order(buf_size);

		pp_params.flags = PP_FLAG_DMA_MAP;

		pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
@@ -1478,6 +1512,21 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, 
			netdev_err(pfvf->netdev, "Creation of page pool failed\n");

			return PTR_ERR(pool->page_pool);

		}

		return 0;

	}



	/* Set XSK pool to support AF_XDP zero-copy */

	xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id);

	if (xsk_pool) {

		pool->xsk_pool = xsk_pool;

		pool->xdp_cnt = numptrs;

		pool->xdp = devm_kcalloc(pfvf->dev,

					 numptrs, sizeof(struct xdp_buff *), GFP_KERNEL);

		if (IS_ERR(pool->xdp)) {

			netdev_err(pfvf->netdev, "Creation of xsk pool failed\n");

			return PTR_ERR(pool->xdp);

		}

	}



	return 0;

}
@@ -1537,9 +1586,18 @@ int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) 
		}



		for (ptr = 0; ptr < num_sqbs; ptr++) {

			err = otx2_alloc_rbuf(pfvf, pool, &bufptr);

			if (err)

			err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);

			if (err) {

				if (pool->xsk_pool) {

					ptr--;

					while (ptr >= 0) {

						xsk_buff_free(pool->xdp[ptr]);

						ptr--;

					}

				}

				goto err_mem;

			}



			pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr);

			sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr;

		}
@@ -1589,11 +1647,19 @@ int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) 
	/* Allocate pointers and free them to aura/pool */

	for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) {

		pool = &pfvf->qset.pool[pool_id];



		for (ptr = 0; ptr < num_ptrs; ptr++) {

			err = otx2_alloc_rbuf(pfvf, pool, &bufptr);

			if (err)

			err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);

			if (err) {

				if (pool->xsk_pool) {

					while (ptr)

						xsk_buff_free(pool->xdp[--ptr]);

				}

				return -ENOMEM;

			}



			pfvf->hw_ops->aura_freeptr(pfvf, pool_id,

						   pool->xsk_pool ? bufptr :

						   bufptr + OTX2_HEAD_ROOM);

		}

	}

drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h

+15 −2
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#include <linux/time64.h>

#include <linux/dim.h>

#include <uapi/linux/if_macsec.h>

#include <net/page_pool/helpers.h>



#include <mbox.h>

#include <npc.h>
@@ -128,6 +129,12 @@ enum otx2_errcodes_re { 
	ERRCODE_IL4_CSUM = 0x22,

};



enum otx2_xdp_action {

	OTX2_XDP_TX	  = BIT(0),

	OTX2_XDP_REDIRECT = BIT(1),

	OTX2_AF_XDP_FRAME = BIT(2),

};



struct otx2_dev_stats {

	u64 rx_bytes;

	u64 rx_frames;
@@ -531,6 +538,8 @@ struct otx2_nic { 


	/* Inline ipsec */

	struct cn10k_ipsec	ipsec;

	/* af_xdp zero-copy */

	unsigned long		*af_xdp_zc_qidx;

};



static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
@@ -1002,7 +1011,7 @@ void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq); 
void otx2_free_pending_sqe(struct otx2_nic *pfvf);

void otx2_sqb_flush(struct otx2_nic *pfvf);

int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,

		    dma_addr_t *dma);

		    dma_addr_t *dma, int qidx, int idx);

int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable);

void otx2_ctx_disable(struct mbox *mbox, int type, bool npa);

int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable);
@@ -1032,6 +1041,8 @@ void otx2_pfaf_mbox_destroy(struct otx2_nic *pf); 
void otx2_disable_mbox_intr(struct otx2_nic *pf);

void otx2_disable_napi(struct otx2_nic *pf);

irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq);

int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura);

int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx);



/* RSS configuration APIs*/

int otx2_rss_init(struct otx2_nic *pfvf);
@@ -1094,7 +1105,8 @@ int otx2_del_macfilter(struct net_device *netdev, const u8 *mac); 
int otx2_add_macfilter(struct net_device *netdev, const u8 *mac);

int otx2_enable_rxvlan(struct otx2_nic *pf, bool enable);

int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf);

bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx);

bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,

			    u64 iova, int len, u16 qidx, u16 flags);

u16 otx2_get_max_mtu(struct otx2_nic *pfvf);

int otx2_handle_ntuple_tc_features(struct net_device *netdev,

				   netdev_features_t features);
@@ -1175,4 +1187,5 @@ static inline int mcam_entry_cmp(const void *a, const void *b) 
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,

				 struct sk_buff *skb, int seg, int *len);

void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg);

int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx);

#endif /* OTX2_COMMON_H */

drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c

+5 −1
Original line number Diff line number Diff line
@@ -910,9 +910,13 @@ static int otx2_get_rxfh(struct net_device *dev, 
		return -ENOENT;



	if (indir) {

		for (idx = 0; idx < rss->rss_size; idx++)

		for (idx = 0; idx < rss->rss_size; idx++) {

			/* Ignore if the rx queue is AF_XDP zero copy enabled */

			if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))

				continue;

			indir[idx] = rss_ctx->ind_tbl[idx];

		}

	}

	if (rxfh->key)

		memcpy(rxfh->key, rss->key, sizeof(rss->key));