Commit 0a8fe916 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch '200GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue 

Tony Nguyen says:

====================
idpf: add XSk support

Alexander Lobakin says:

Add support for XSk xmit and receive using libeth_xdp.

This includes adding interfaces to reconfigure/enable/disable only
a particular set of queues and support for checksum offload XSk Tx
metadata.
libeth_xdp's implementation mostly matches the one of ice: batched
allocations and sending, unrolled descriptor writes etc. But unlike
other Intel drivers, XSk wakeup is implemented using CSD/IPI instead
of HW "software interrupt". In lots of different tests, this yielded
way better perf than SW interrupts, but also, this gives better
control over which CPU will handle the NAPI loop (SW interrupts are
a subject to irqbalance and stuff, while CSDs are strictly pinned
1:1 to the core of the same index).
Note that the header split is always disabled for XSk queues, as
for now we see no reasons to have it there.

XSk xmit perf is up to 3x comparing to ice. XSk XDP_PASS is also
faster a bunch as it uses system percpu page_pools, so that the
only overhead left is memcpy(). The rest is at least comparable.

* '200GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue:
  idpf: enable XSk features and ndo_xsk_wakeup
  idpf: implement Rx path for AF_XDP
  idpf: implement XSk xmit
  idpf: add XSk pool initialization
  idpf: add virtchnl functions to manage selected queues
====================

Link: https://patch.msgid.link/20250924175230.1290529-1-anthony.l.nguyen@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 0e41b0af 96da9d67

drivers/net/ethernet/intel/idpf/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -23,3 +23,4 @@ idpf-$(CONFIG_PTP_1588_CLOCK) += idpf_ptp.o 
idpf-$(CONFIG_PTP_1588_CLOCK)	+= idpf_virtchnl_ptp.o



idpf-y				+= xdp.o

idpf-y				+= xsk.o

drivers/net/ethernet/intel/idpf/idpf.h

+7 −0
Original line number Diff line number Diff line
@@ -995,6 +995,13 @@ static inline void idpf_vport_ctrl_unlock(struct net_device *netdev) 
	mutex_unlock(&np->adapter->vport_ctrl_lock);

}



static inline bool idpf_vport_ctrl_is_locked(struct net_device *netdev)

{

	struct idpf_netdev_priv *np = netdev_priv(netdev);



	return mutex_is_locked(&np->adapter->vport_ctrl_lock);

}



void idpf_statistics_task(struct work_struct *work);

void idpf_init_task(struct work_struct *work);

void idpf_service_task(struct work_struct *work);

drivers/net/ethernet/intel/idpf/idpf_ethtool.c

+4 −4
Original line number Diff line number Diff line
@@ -1245,8 +1245,8 @@ static void idpf_get_ethtool_stats(struct net_device *netdev, 
 *

 * returns pointer to rx vector

 */

static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport,

					       int q_num)

struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport,

					u32 q_num)

{

	int q_grp, q_idx;
@@ -1266,8 +1266,8 @@ static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, 
 *

 * returns pointer to tx vector

 */

static struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport,

					       int q_num)

struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport,

					u32 q_num)

{

	int q_grp;

drivers/net/ethernet/intel/idpf/idpf_lib.c

+6 −4
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#include "idpf_virtchnl.h"

#include "idpf_ptp.h"

#include "xdp.h"

#include "xsk.h"



static const struct net_device_ops idpf_netdev_ops;
@@ -1424,16 +1425,16 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) 
		goto queues_rel;

	}



	err = idpf_rx_bufs_init_all(vport);

	err = idpf_queue_reg_init(vport);

	if (err) {

		dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n",

		dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n",

			vport->vport_id, err);

		goto queues_rel;

	}



	err = idpf_queue_reg_init(vport);

	err = idpf_rx_bufs_init_all(vport);

	if (err) {

		dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n",

		dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n",

			vport->vport_id, err);

		goto queues_rel;

	}
@@ -2618,4 +2619,5 @@ static const struct net_device_ops idpf_netdev_ops = { 
	.ndo_hwtstamp_set = idpf_hwtstamp_set,

	.ndo_bpf = idpf_xdp,

	.ndo_xdp_xmit = idpf_xdp_xmit,

	.ndo_xsk_wakeup = idpf_xsk_wakeup,

};

drivers/net/ethernet/intel/idpf/idpf_txrx.c

+427 −24
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#include "idpf_ptp.h"

#include "idpf_virtchnl.h"

#include "xdp.h"

#include "xsk.h"



#define idpf_tx_buf_next(buf)		(*(u32 *)&(buf)->priv)

LIBETH_SQE_CHECK_PRIV(u32);
@@ -53,11 +54,7 @@ void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 
	}

}



/**

 * idpf_tx_buf_rel_all - Free any empty Tx buffers

 * @txq: queue to be cleaned

 */

static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq)

static void idpf_tx_buf_clean(struct idpf_tx_queue *txq)

{

	struct libeth_sq_napi_stats ss = { };

	struct xdp_frame_bulk bq;
@@ -66,19 +63,30 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) 
		.bq	= &bq,

		.ss	= &ss,

	};

	u32 i;



	/* Buffers already cleared, nothing to do */

	if (!txq->tx_buf)

		return;



	xdp_frame_bulk_init(&bq);



	/* Free all the Tx buffer sk_buffs */

	for (i = 0; i < txq->buf_pool_size; i++)

	for (u32 i = 0; i < txq->buf_pool_size; i++)

		libeth_tx_complete_any(&txq->tx_buf[i], &cp);



	xdp_flush_frame_bulk(&bq);

}



/**

 * idpf_tx_buf_rel_all - Free any empty Tx buffers

 * @txq: queue to be cleaned

 */

static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq)

{

	/* Buffers already cleared, nothing to do */

	if (!txq->tx_buf)

		return;



	if (idpf_queue_has(XSK, txq))

		idpf_xsksq_clean(txq);

	else

		idpf_tx_buf_clean(txq);



	kfree(txq->tx_buf);

	txq->tx_buf = NULL;
@@ -102,6 +110,8 @@ static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) 
	if (!xdp)

		netdev_tx_reset_subqueue(txq->netdev, txq->idx);



	idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX);



	if (!txq->desc_ring)

		return;
@@ -122,6 +132,8 @@ static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) 
 */

static void idpf_compl_desc_rel(struct idpf_compl_queue *complq)

{

	idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION);



	if (!complq->comp)

		return;
@@ -214,6 +226,8 @@ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, 
	tx_q->next_to_clean = 0;

	idpf_queue_set(GEN_CHK, tx_q);



	idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX);



	if (!idpf_queue_has(FLOW_SCH_EN, tx_q))

		return 0;
@@ -273,6 +287,9 @@ static int idpf_compl_desc_alloc(const struct idpf_vport *vport, 
	complq->next_to_clean = 0;

	idpf_queue_set(GEN_CHK, complq);



	idpf_xsk_setup_queue(vport, complq,

			     VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION);



	return 0;

}
@@ -372,6 +389,11 @@ static void idpf_rx_buf_rel_bufq(struct idpf_buf_queue *bufq) 
	if (!bufq->buf)

		return;



	if (idpf_queue_has(XSK, bufq)) {

		idpf_xskfq_rel(bufq);

		return;

	}



	/* Free all the bufs allocated and given to hw on Rx queue */

	for (u32 i = 0; i < bufq->desc_count; i++)

		idpf_rx_page_rel(&bufq->buf[i]);
@@ -420,11 +442,14 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, 
	if (!rxq)

		return;



	if (!idpf_queue_has(XSK, rxq))

		libeth_xdp_return_stash(&rxq->xdp);



	if (!idpf_is_queue_model_split(model))

		idpf_rx_buf_rel_all(rxq);



	idpf_xsk_clear_queue(rxq, VIRTCHNL2_QUEUE_TYPE_RX);



	rxq->next_to_alloc = 0;

	rxq->next_to_clean = 0;

	rxq->next_to_use = 0;
@@ -447,6 +472,7 @@ static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq, 
		return;



	idpf_rx_buf_rel_bufq(bufq);

	idpf_xsk_clear_queue(bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER);



	bufq->next_to_alloc = 0;

	bufq->next_to_clean = 0;
@@ -734,6 +760,9 @@ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, 
	};

	int ret;



	if (idpf_queue_has(XSK, bufq))

		return idpf_xskfq_init(bufq);



	ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi);

	if (ret)

		return ret;
@@ -829,6 +858,8 @@ static int idpf_rx_desc_alloc(const struct idpf_vport *vport, 
	rxq->next_to_use = 0;

	idpf_queue_set(GEN_CHK, rxq);



	idpf_xsk_setup_queue(vport, rxq, VIRTCHNL2_QUEUE_TYPE_RX);



	return 0;

}
@@ -854,9 +885,10 @@ static int idpf_bufq_desc_alloc(const struct idpf_vport *vport, 
	bufq->next_to_alloc = 0;

	bufq->next_to_clean = 0;

	bufq->next_to_use = 0;



	idpf_queue_set(GEN_CHK, bufq);



	idpf_xsk_setup_queue(vport, bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER);



	return 0;

}
@@ -922,6 +954,341 @@ static int idpf_rx_desc_alloc_all(struct idpf_vport *vport) 
	return err;

}



static int idpf_init_queue_set(const struct idpf_queue_set *qs)

{

	const struct idpf_vport *vport = qs->vport;

	bool splitq;

	int err;



	splitq = idpf_is_queue_model_split(vport->rxq_model);



	for (u32 i = 0; i < qs->num; i++) {

		const struct idpf_queue_ptr *q = &qs->qs[i];

		struct idpf_buf_queue *bufq;



		switch (q->type) {

		case VIRTCHNL2_QUEUE_TYPE_RX:

			err = idpf_rx_desc_alloc(vport, q->rxq);

			if (err)

				break;



			err = idpf_xdp_rxq_info_init(q->rxq);

			if (err)

				break;



			if (!splitq)

				err = idpf_rx_bufs_init_singleq(q->rxq);



			break;

		case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER:

			bufq = q->bufq;



			err = idpf_bufq_desc_alloc(vport, bufq);

			if (err)

				break;



			for (u32 j = 0; j < bufq->q_vector->num_bufq; j++) {

				struct idpf_buf_queue * const *bufqs;

				enum libeth_fqe_type type;

				u32 ts;



				bufqs = bufq->q_vector->bufq;

				if (bufqs[j] != bufq)

					continue;



				if (j) {

					type = LIBETH_FQE_SHORT;

					ts = bufqs[j - 1]->truesize >> 1;

				} else {

					type = LIBETH_FQE_MTU;

					ts = 0;

				}



				bufq->truesize = ts;



				err = idpf_rx_bufs_init(bufq, type);

				break;

			}



			break;

		case VIRTCHNL2_QUEUE_TYPE_TX:

			err = idpf_tx_desc_alloc(vport, q->txq);

			break;

		case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:

			err = idpf_compl_desc_alloc(vport, q->complq);

			break;

		default:

			continue;

		}



		if (err)

			return err;

	}



	return 0;

}



static void idpf_clean_queue_set(const struct idpf_queue_set *qs)

{

	const struct idpf_vport *vport = qs->vport;

	struct device *dev = vport->netdev->dev.parent;



	for (u32 i = 0; i < qs->num; i++) {

		const struct idpf_queue_ptr *q = &qs->qs[i];



		switch (q->type) {

		case VIRTCHNL2_QUEUE_TYPE_RX:

			idpf_xdp_rxq_info_deinit(q->rxq, vport->rxq_model);

			idpf_rx_desc_rel(q->rxq, dev, vport->rxq_model);

			break;

		case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER:

			idpf_rx_desc_rel_bufq(q->bufq, dev);

			break;

		case VIRTCHNL2_QUEUE_TYPE_TX:

			idpf_tx_desc_rel(q->txq);



			if (idpf_queue_has(XDP, q->txq)) {

				q->txq->pending = 0;

				q->txq->xdp_tx = 0;

			} else {

				q->txq->txq_grp->num_completions_pending = 0;

			}



			writel(q->txq->next_to_use, q->txq->tail);

			break;

		case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:

			idpf_compl_desc_rel(q->complq);

			q->complq->num_completions = 0;

			break;

		default:

			break;

		}

	}

}



static void idpf_qvec_ena_irq(struct idpf_q_vector *qv)

{

	if (qv->num_txq) {

		u32 itr;



		if (IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode))

			itr = qv->vport->tx_itr_profile[qv->tx_dim.profile_ix];

		else

			itr = qv->tx_itr_value;



		idpf_vport_intr_write_itr(qv, itr, true);

	}



	if (qv->num_rxq) {

		u32 itr;



		if (IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode))

			itr = qv->vport->rx_itr_profile[qv->rx_dim.profile_ix];

		else

			itr = qv->rx_itr_value;



		idpf_vport_intr_write_itr(qv, itr, false);

	}



	if (qv->num_txq || qv->num_rxq)

		idpf_vport_intr_update_itr_ena_irq(qv);

}



/**

 * idpf_vector_to_queue_set - create a queue set associated with the given

 *			      queue vector

 * @qv: queue vector corresponding to the queue pair

 *

 * Returns a pointer to a dynamically allocated array of pointers to all

 * queues associated with a given queue vector (@qv).

 * Please note that the caller is responsible to free the memory allocated

 * by this function using kfree().

 *

 * Return: &idpf_queue_set on success, %NULL in case of error.

 */

static struct idpf_queue_set *

idpf_vector_to_queue_set(struct idpf_q_vector *qv)

{

	bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq;

	struct idpf_vport *vport = qv->vport;

	struct idpf_queue_set *qs;

	u32 num;



	num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq;

	num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2;

	if (!num)

		return NULL;



	qs = idpf_alloc_queue_set(vport, num);

	if (!qs)

		return NULL;



	num = 0;



	for (u32 i = 0; i < qv->num_bufq; i++) {

		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER;

		qs->qs[num++].bufq = qv->bufq[i];

	}



	for (u32 i = 0; i < qv->num_rxq; i++) {

		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX;

		qs->qs[num++].rxq = qv->rx[i];

	}



	for (u32 i = 0; i < qv->num_txq; i++) {

		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;

		qs->qs[num++].txq = qv->tx[i];

	}



	for (u32 i = 0; i < qv->num_complq; i++) {

		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;

		qs->qs[num++].complq = qv->complq[i];

	}



	if (!vport->xdp_txq_offset)

		goto finalize;



	if (xdp) {

		for (u32 i = 0; i < qv->num_rxq; i++) {

			u32 idx = vport->xdp_txq_offset + qv->rx[i]->idx;



			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;

			qs->qs[num++].txq = vport->txqs[idx];



			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;

			qs->qs[num++].complq = vport->txqs[idx]->complq;

		}

	} else {

		for (u32 i = 0; i < qv->num_xsksq; i++) {

			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;

			qs->qs[num++].txq = qv->xsksq[i];



			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;

			qs->qs[num++].complq = qv->xsksq[i]->complq;

		}

	}



finalize:

	if (num != qs->num) {

		kfree(qs);

		return NULL;

	}



	return qs;

}



static int idpf_qp_enable(const struct idpf_queue_set *qs, u32 qid)

{

	struct idpf_vport *vport = qs->vport;

	struct idpf_q_vector *q_vector;

	int err;



	q_vector = idpf_find_rxq_vec(vport, qid);



	err = idpf_init_queue_set(qs);

	if (err) {

		netdev_err(vport->netdev, "Could not initialize queues in pair %u: %pe\n",

			   qid, ERR_PTR(err));

		return err;

	}



	if (!vport->xdp_txq_offset)

		goto config;



	q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp,

					       vport->num_q_vectors),

				  sizeof(*q_vector->xsksq), GFP_KERNEL);

	if (!q_vector->xsksq)

		return -ENOMEM;



	for (u32 i = 0; i < qs->num; i++) {

		const struct idpf_queue_ptr *q = &qs->qs[i];



		if (q->type != VIRTCHNL2_QUEUE_TYPE_TX)

			continue;



		if (!idpf_queue_has(XSK, q->txq))

			continue;



		idpf_xsk_init_wakeup(q_vector);



		q->txq->q_vector = q_vector;

		q_vector->xsksq[q_vector->num_xsksq++] = q->txq;

	}



config:

	err = idpf_send_config_queue_set_msg(qs);

	if (err) {

		netdev_err(vport->netdev, "Could not configure queues in pair %u: %pe\n",

			   qid, ERR_PTR(err));

		return err;

	}



	err = idpf_send_enable_queue_set_msg(qs);

	if (err) {

		netdev_err(vport->netdev, "Could not enable queues in pair %u: %pe\n",

			   qid, ERR_PTR(err));

		return err;

	}



	napi_enable(&q_vector->napi);

	idpf_qvec_ena_irq(q_vector);



	netif_start_subqueue(vport->netdev, qid);



	return 0;

}



static int idpf_qp_disable(const struct idpf_queue_set *qs, u32 qid)

{

	struct idpf_vport *vport = qs->vport;

	struct idpf_q_vector *q_vector;

	int err;



	q_vector = idpf_find_rxq_vec(vport, qid);

	netif_stop_subqueue(vport->netdev, qid);



	writel(0, q_vector->intr_reg.dyn_ctl);

	napi_disable(&q_vector->napi);



	err = idpf_send_disable_queue_set_msg(qs);

	if (err) {

		netdev_err(vport->netdev, "Could not disable queues in pair %u: %pe\n",

			   qid, ERR_PTR(err));

		return err;

	}



	idpf_clean_queue_set(qs);



	kfree(q_vector->xsksq);

	q_vector->num_xsksq = 0;



	return 0;

}



/**

 * idpf_qp_switch - enable or disable queues associated with queue pair

 * @vport: vport to switch the pair for

 * @qid: index of the queue pair to switch

 * @en: whether to enable or disable the pair

 *

 * Return: 0 on success, -errno on failure.

 */

int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en)

{

	struct idpf_q_vector *q_vector = idpf_find_rxq_vec(vport, qid);

	struct idpf_queue_set *qs __free(kfree) = NULL;



	if (idpf_find_txq_vec(vport, qid) != q_vector)

		return -EINVAL;



	qs = idpf_vector_to_queue_set(q_vector);

	if (!qs)

		return -ENOMEM;



	return en ? idpf_qp_enable(qs, qid) : idpf_qp_disable(qs, qid);

}



/**

 * idpf_txq_group_rel - Release all resources for txq groups

 * @vport: vport to release txq groups on
@@ -1365,6 +1732,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) 
			q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter);

			q->netdev = vport->netdev;

			q->txq_grp = tx_qgrp;

			q->rel_q_id = j;



			if (!split) {

				q->clean_budget = vport->compln_clean_budget;
@@ -3030,7 +3398,7 @@ __idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, 
	return 0;

}



static bool idpf_rx_process_skb_fields(struct sk_buff *skb,

bool idpf_rx_process_skb_fields(struct sk_buff *skb,

				const struct libeth_xdp_buff *xdp,

				struct libeth_rq_napi_stats *rs)

{
@@ -3390,7 +3758,7 @@ static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, 
	struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data;



	q_vector->total_events++;

	napi_schedule(&q_vector->napi);

	napi_schedule_irqoff(&q_vector->napi);



	return IRQ_HANDLED;

}
@@ -3431,6 +3799,8 @@ void idpf_vport_intr_rel(struct idpf_vport *vport) 
	for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {

		struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];



		kfree(q_vector->xsksq);

		q_vector->xsksq = NULL;

		kfree(q_vector->complq);

		q_vector->complq = NULL;

		kfree(q_vector->bufq);
@@ -3889,7 +4259,9 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, 
		struct idpf_rx_queue *rxq = q_vec->rx[i];

		int pkts_cleaned_per_q;



		pkts_cleaned_per_q = idpf_rx_splitq_clean(rxq, budget_per_q);

		pkts_cleaned_per_q = idpf_queue_has(XSK, rxq) ?

				     idpf_xskrq_poll(rxq, budget_per_q) :

				     idpf_rx_splitq_clean(rxq, budget_per_q);

		/* if we clean as many as budgeted, we must not be done */

		if (pkts_cleaned_per_q >= budget_per_q)

			clean_complete = false;
@@ -3899,8 +4271,10 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, 


	nid = numa_mem_id();



	for (i = 0; i < q_vec->num_bufq; i++)

	for (i = 0; i < q_vec->num_bufq; i++) {

		if (!idpf_queue_has(XSK, q_vec->bufq[i]))

			idpf_rx_clean_refillq_all(q_vec->bufq[i], nid);

	}



	return clean_complete;

}
@@ -3914,7 +4288,7 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) 
{

	struct idpf_q_vector *q_vector =

				container_of(napi, struct idpf_q_vector, napi);

	bool clean_complete;

	bool clean_complete = true;

	int work_done = 0;



	/* Handle case where we are called by netpoll with a budget of 0 */
@@ -3924,8 +4298,13 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) 
		return 0;

	}



	clean_complete = idpf_rx_splitq_clean_all(q_vector, budget, &work_done);

	clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done);

	for (u32 i = 0; i < q_vector->num_xsksq; i++)

		clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]);



	clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget,

						   &work_done);

	clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget,

						   &work_done);



	/* If work not completed, return budget and polling will return */

	if (!clean_complete) {
@@ -3938,7 +4317,7 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) 
	/* Exit the polling mode, but don't re-enable interrupts if stack might

	 * poll us due to busy-polling

	 */

	if (likely(napi_complete_done(napi, work_done)))

	if (napi_complete_done(napi, work_done))

		idpf_vport_intr_update_itr_ena_irq(q_vector);

	else

		idpf_vport_intr_set_wb_on_itr(q_vector);
@@ -4031,6 +4410,21 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) 


		qv_idx++;

	}



	for (i = 0; i < vport->num_xdp_txq; i++) {

		struct idpf_tx_queue *xdpsq;

		struct idpf_q_vector *qv;



		xdpsq = vport->txqs[vport->xdp_txq_offset + i];

		if (!idpf_queue_has(XSK, xdpsq))

			continue;



		qv = idpf_find_rxq_vec(vport, i);

		idpf_xsk_init_wakeup(qv);



		xdpsq->q_vector = qv;

		qv->xsksq[qv->num_xsksq++] = xdpsq;

	}

}



/**
@@ -4168,6 +4562,15 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport) 
					   GFP_KERNEL);

		if (!q_vector->complq)

			goto error;



		if (!vport->xdp_txq_offset)

			continue;



		q_vector->xsksq = kcalloc(rxqs_per_vector,

					  sizeof(*q_vector->xsksq),

					  GFP_KERNEL);

		if (!q_vector->xsksq)

			goto error;

	}



	return 0;