Commit 2773cb0b authored by Eric Dumazet's avatar Eric Dumazet Committed by Paolo Abeni
Browse files

net_sched: use qdisc_skb_cb(skb)->pkt_segs in bstats_update() 



Avoid up to two cache line misses in qdisc dequeue() to fetch
skb_shinfo(skb)->gso_segs/gso_size while qdisc spinlock is held.

This gives a 5 % improvement in a TX intensive workload.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20251121083256.674562-6-edumazet@google.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent f9e00e51

include/net/sch_generic.h

+10 −3
Original line number Diff line number Diff line
@@ -829,6 +829,15 @@ static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 
	return qdisc_skb_cb(skb)->pkt_len;

}



static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb)

{

	u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;



	DEBUG_NET_WARN_ON_ONCE(pkt_segs !=

			(skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1));

	return pkt_segs;

}



/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */

enum net_xmit_qdisc_t {

	__NET_XMIT_STOLEN = 0x00010000,
@@ -870,9 +879,7 @@ static inline void _bstats_update(struct gnet_stats_basic_sync *bstats, 
static inline void bstats_update(struct gnet_stats_basic_sync *bstats,

				 const struct sk_buff *skb)

{

	_bstats_update(bstats,

		       qdisc_pkt_len(skb),

		       skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1);

	_bstats_update(bstats, qdisc_pkt_len(skb), qdisc_pkt_segs(skb));

}



static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,

net/sched/sch_cake.c

+1 −0
Original line number Diff line number Diff line
@@ -1800,6 +1800,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, 
		skb_list_walk_safe(segs, segs, nskb) {

			skb_mark_not_on_list(segs);

			qdisc_skb_cb(segs)->pkt_len = segs->len;

			qdisc_skb_cb(segs)->pkt_segs = 1;

			cobalt_set_enqueue_time(segs, now);

			get_cobalt_cb(segs)->adjusted_len = cake_overhead(q,

									  segs);

net/sched/sch_dualpi2.c

+1 −0
Original line number Diff line number Diff line
@@ -475,6 +475,7 @@ static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, 
			 * (3) Enqueue fragment & set ts in dualpi2_enqueue_skb

			 */

			qdisc_skb_cb(nskb)->pkt_len = nskb->len;

			qdisc_skb_cb(nskb)->pkt_segs = 1;

			dualpi2_skb_cb(nskb)->classified =

				dualpi2_skb_cb(skb)->classified;

			dualpi2_skb_cb(nskb)->ect = dualpi2_skb_cb(skb)->ect;

net/sched/sch_netem.c

+1 −0
Original line number Diff line number Diff line
@@ -429,6 +429,7 @@ static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch, 
	struct sk_buff *segs;

	netdev_features_t features = netif_skb_features(skb);



	qdisc_skb_cb(skb)->pkt_segs = 1;

	segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);



	if (IS_ERR_OR_NULL(segs)) {

net/sched/sch_qfq.c

+1 −1
Original line number Diff line number Diff line
@@ -1250,7 +1250,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, 
		}

	}



	gso_segs = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;

	gso_segs = qdisc_pkt_segs(skb);

	err = qdisc_enqueue(skb, cl->qdisc, to_free);

	if (unlikely(err != NET_XMIT_SUCCESS)) {

		pr_debug("qfq_enqueue: enqueue failed %d\n", err);