Commit 3c0bf13f authored by Brett Creeley's avatar Brett Creeley Committed by Jakub Kicinski
Browse files

ionic: Allow XDP program to be hot swapped 



Using examples of other driver(s), add the ability to hot-swap an XDP
program without having to reconfigure the queues. To prevent the
q->xdp_prog to be read/written more than once use READ_ONCE() and
WRITE_ONCE() on the q->xdp_prog.

The q->xdp_prog was being checked in multiple different for loops in the
hot path. The change to allow xdp_prog hot swapping created the
possibility for many READ_ONCE(q->xdp_prog) calls during a single napi
callback. Refactor the Rx napi handling to allow a previous
READ_ONCE(q->xdp_prog) (or NULL for hwstamp_rxq) to be passed into the
relevant functions.

Also, move other Rx related hotpath handling into the newly created
ionic_rx_cq_service() function to reduce the scope of the xdp_prog
local variable and put all Rx handling in one function similar to Tx.

Signed-off-by: default avatarBrett Creeley <brett.creeley@amd.com>
Signed-off-by: default avatarShannon Nelson <shannon.nelson@amd.com>
Link: https://patch.msgid.link/20240906232623.39651-8-brett.creeley@amd.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent ac8813c0

drivers/net/ethernet/pensando/ionic/ionic_lif.c

+8 −4
Original line number Diff line number Diff line
@@ -1074,7 +1074,7 @@ int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif) 
			goto err_qcq_init;



		if (test_bit(IONIC_LIF_F_UP, lif->state)) {

			ionic_rx_fill(&rxq->q);

			ionic_rx_fill(&rxq->q, NULL);

			err = ionic_qcq_enable(rxq);

			if (err)

				goto err_qcq_enable;
@@ -2190,7 +2190,8 @@ static int ionic_txrx_enable(struct ionic_lif *lif) 
			goto err_out;

		}



		ionic_rx_fill(&lif->rxqcqs[i]->q);

		ionic_rx_fill(&lif->rxqcqs[i]->q,

			      READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));

		err = ionic_qcq_enable(lif->rxqcqs[i]);

		if (err)

			goto err_out;
@@ -2203,7 +2204,7 @@ static int ionic_txrx_enable(struct ionic_lif *lif) 
	}



	if (lif->hwstamp_rxq) {

		ionic_rx_fill(&lif->hwstamp_rxq->q);

		ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);

		err = ionic_qcq_enable(lif->hwstamp_rxq);

		if (err)

			goto err_out_hwstamp_rx;
@@ -2746,7 +2747,7 @@ static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif) 
	for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {

		struct ionic_queue *q = &lif->rxqcqs[i]->q;



		q->xdp_prog = xdp_prog;

		WRITE_ONCE(q->xdp_prog, xdp_prog);

	}

}
@@ -2777,6 +2778,9 @@ static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf) 


	if (!netif_running(netdev)) {

		old_prog = xchg(&lif->xdp_prog, bpf->prog);

	} else if (lif->xdp_prog && bpf->prog) {

		old_prog = xchg(&lif->xdp_prog, bpf->prog);

		ionic_xdp_rxqs_prog_update(lif);

	} else {

		struct ionic_queue_params qparams;

drivers/net/ethernet/pensando/ionic/ionic_txrx.c

+41 −17
Original line number Diff line number Diff line
@@ -602,7 +602,8 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, 


static void ionic_rx_clean(struct ionic_queue *q,

			   struct ionic_rx_desc_info *desc_info,

			   struct ionic_rxq_comp *comp)

			   struct ionic_rxq_comp *comp,

			   struct bpf_prog *xdp_prog)

{

	struct net_device *netdev = q->lif->netdev;

	struct ionic_qcq *qcq = q_to_qcq(q);
@@ -631,8 +632,8 @@ static void ionic_rx_clean(struct ionic_queue *q, 
	stats->pkts++;

	stats->bytes += len;



	if (q->xdp_prog) {

		if (ionic_run_xdp(stats, netdev, q->xdp_prog, q, desc_info->bufs, len))

	if (xdp_prog) {

		if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len))

			return;

		synced = true;

		headroom = XDP_PACKET_HEADROOM;
@@ -718,7 +719,7 @@ static void ionic_rx_clean(struct ionic_queue *q, 
		napi_gro_frags(&qcq->napi);

}



bool ionic_rx_service(struct ionic_cq *cq)

static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog)

{

	struct ionic_rx_desc_info *desc_info;

	struct ionic_queue *q = cq->bound_q;
@@ -740,11 +741,16 @@ bool ionic_rx_service(struct ionic_cq *cq) 
	q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);



	/* clean the related q entry, only one per qc completion */

	ionic_rx_clean(q, desc_info, comp);

	ionic_rx_clean(q, desc_info, comp, xdp_prog);



	return true;

}



bool ionic_rx_service(struct ionic_cq *cq)

{

	return __ionic_rx_service(cq, NULL);

}



static inline void ionic_write_cmb_desc(struct ionic_queue *q,

					void *desc)

{
@@ -755,7 +761,7 @@ static inline void ionic_write_cmb_desc(struct ionic_queue *q, 
		memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0]));

}



void ionic_rx_fill(struct ionic_queue *q)

void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog)

{

	struct net_device *netdev = q->lif->netdev;

	struct ionic_rx_desc_info *desc_info;
@@ -783,7 +789,7 @@ void ionic_rx_fill(struct ionic_queue *q) 


	len = netdev->mtu + VLAN_ETH_HLEN;



	if (q->xdp_prog) {

	if (xdp_prog) {

		/* Always alloc the full size buffer, but only need

		 * the actual frag_len in the descriptor

		 * XDP uses space in the first buffer, so account for
@@ -964,6 +970,32 @@ static void ionic_xdp_do_flush(struct ionic_cq *cq) 
	}

}



static unsigned int ionic_rx_cq_service(struct ionic_cq *cq,

					unsigned int work_to_do)

{

	struct ionic_queue *q = cq->bound_q;

	unsigned int work_done = 0;

	struct bpf_prog *xdp_prog;



	if (work_to_do == 0)

		return 0;



	xdp_prog = READ_ONCE(q->xdp_prog);

	while (__ionic_rx_service(cq, xdp_prog)) {

		if (cq->tail_idx == cq->num_descs - 1)

			cq->done_color = !cq->done_color;



		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);



		if (++work_done >= work_to_do)

			break;

	}

	ionic_rx_fill(q, xdp_prog);

	ionic_xdp_do_flush(cq);



	return work_done;

}



int ionic_rx_napi(struct napi_struct *napi, int budget)

{

	struct ionic_qcq *qcq = napi_to_qcq(napi);
@@ -974,12 +1006,8 @@ int ionic_rx_napi(struct napi_struct *napi, int budget) 
	if (unlikely(!budget))

		return budget;



	work_done = ionic_cq_service(cq, budget,

				     ionic_rx_service, NULL, NULL);

	work_done = ionic_rx_cq_service(cq, budget);



	ionic_rx_fill(cq->bound_q);



	ionic_xdp_do_flush(cq);

	if (work_done < budget && napi_complete_done(napi, work_done)) {

		ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);

		flags |= IONIC_INTR_CRED_UNMASK;
@@ -1020,12 +1048,8 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) 
	if (unlikely(!budget))

		return budget;



	rx_work_done = ionic_cq_service(rxcq, budget,

					ionic_rx_service, NULL, NULL);



	ionic_rx_fill(rxcq->bound_q);

	rx_work_done = ionic_rx_cq_service(rxcq, budget);



	ionic_xdp_do_flush(rxcq);

	if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {

		ionic_dim_update(rxqcq, 0);

		flags |= IONIC_INTR_CRED_UNMASK;

drivers/net/ethernet/pensando/ionic/ionic_txrx.h

+3 −1
Original line number Diff line number Diff line
@@ -4,9 +4,11 @@ 
#ifndef _IONIC_TXRX_H_

#define _IONIC_TXRX_H_



struct bpf_prog;



void ionic_tx_flush(struct ionic_cq *cq);



void ionic_rx_fill(struct ionic_queue *q);

void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog);

void ionic_rx_empty(struct ionic_queue *q);

void ionic_tx_empty(struct ionic_queue *q);

int ionic_rx_napi(struct napi_struct *napi, int budget);