Commit 7a4cd71f authored by Eric Dumazet's avatar Eric Dumazet Committed by Paolo Abeni
Browse files

net: add vlan_get_protocol_offset_inline() helper 



skb_protocol() is bloated, and forces slow stack canaries in many
fast paths.

Add vlan_get_protocol_offset_inline() which deals with the non-vlan
common cases.

__vlan_get_protocol_offset() is now out of line.

It returns a vlan_type_depth struct to avoid stack canaries in callers.

struct vlan_type_depth {
       __be16 type;
       u16 depth;
};

$ scripts/bloat-o-meter -t vmlinux.old vmlinux.new
add/remove: 0/2 grow/shrink: 0/22 up/down: 0/-6320 (-6320)
Function                                     old     new   delta
vlan_get_protocol_dgram                       61      59      -2
__pfx_skb_protocol                            16       -     -16
__vlan_get_protocol_offset                   307     273     -34
tap_get_user                                1374    1207    -167
ip_md_tunnel_xmit                           1625    1452    -173
tap_sendmsg                                  940     753    -187
netif_skb_features                          1079     866    -213
netem_enqueue                               3017    2800    -217
vlan_parse_protocol                          271      50    -221
tso_start                                    567     344    -223
fq_dequeue                                  1908    1685    -223
skb_network_protocol                         434     205    -229
ip6_tnl_xmit                                2639    2409    -230
br_dev_queue_push_xmit                       474     236    -238
skb_protocol                                 258       -    -258
packet_parse_headers                         621     357    -264
__ip6_tnl_rcv                               1306    1039    -267
skb_csum_hwoffload_help                      515     224    -291
ip_tunnel_xmit                              2635    2339    -296
sch_frag_xmit_hook                          1582    1233    -349
bpf_skb_ecn_set_ce                           868     457    -411
IP6_ECN_decapsulate                         1297     768    -529
ip_tunnel_rcv                               2121    1489    -632
ipip6_rcv                                   2572    1922    -650
Total: Before=24892803, After=24886483, chg -0.03%

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Reviewed-by: default avatarSimon Horman <horms@kernel.org>
Link: https://patch.msgid.link/20260204053023.1622775-1-edumazet@google.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parent 770e1126

include/linux/if_vlan.h

+22 −29
Original line number Diff line number Diff line
@@ -594,8 +594,17 @@ static inline int vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci) 
	}

}



struct vlan_type_depth {

	__be16 type;

	u16 depth;

};



struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb,

						  __be16 type,

						  int mac_offset);



/**

 * __vlan_get_protocol_offset() - get protocol EtherType.

 * vlan_get_protocol_offset_inline() - get protocol EtherType.

 * @skb: skbuff to query

 * @type: first vlan protocol

 * @mac_offset: MAC offset
@@ -604,40 +613,24 @@ static inline int vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci) 
 * Returns: the EtherType of the packet, regardless of whether it is

 * vlan encapsulated (normal or hardware accelerated) or not.

 */

static inline __be16 __vlan_get_protocol_offset(const struct sk_buff *skb,

static inline

__be16 vlan_get_protocol_offset_inline(const struct sk_buff *skb,

				       __be16 type,

				       int mac_offset,

				       int *depth)

{

	unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;



	/* if type is 802.1Q/AD then the header should already be

	 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at

	 * ETH_HLEN otherwise

	 */

	if (eth_type_vlan(type)) {

		if (vlan_depth) {

			if (WARN_ON(vlan_depth < VLAN_HLEN))

				return 0;

			vlan_depth -= VLAN_HLEN;

		} else {

			vlan_depth = ETH_HLEN;

		}

		do {

			struct vlan_hdr vhdr, *vh;

		struct vlan_type_depth res;



			vh = skb_header_pointer(skb, mac_offset + vlan_depth,

						sizeof(vhdr), &vhdr);

			if (unlikely(!vh || !--parse_depth))

				return 0;

		res = __vlan_get_protocol_offset(skb, type, mac_offset);



			type = vh->h_vlan_encapsulated_proto;

			vlan_depth += VLAN_HLEN;

		} while (eth_type_vlan(type));

		if (depth && res.type)

			*depth = res.depth;

		return res.type;

	}



	if (depth)

		*depth = vlan_depth;

		*depth = skb->mac_len;



	return type;

}
@@ -645,7 +638,7 @@ static inline __be16 __vlan_get_protocol_offset(const struct sk_buff *skb, 
static inline __be16 __vlan_get_protocol(const struct sk_buff *skb, __be16 type,

					 int *depth)

{

	return __vlan_get_protocol_offset(skb, type, 0, depth);

	return vlan_get_protocol_offset_inline(skb, type, 0, depth);

}



/**

net/core/skbuff.c

+36 −0
Original line number Diff line number Diff line
@@ -7444,3 +7444,39 @@ void __put_netmem(netmem_ref netmem) 
		net_devmem_put_net_iov(netmem_to_net_iov(netmem));

}

EXPORT_SYMBOL(__put_netmem);



struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb,

						  __be16 type,

						  int mac_offset)

{

	unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;



	/* if type is 802.1Q/AD then the header should already be

	 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at

	 * ETH_HLEN otherwise

	 */

	if (vlan_depth) {

		if (WARN_ON_ONCE(vlan_depth < VLAN_HLEN))

			return (struct vlan_type_depth) { 0 };

		vlan_depth -= VLAN_HLEN;

	} else {

		vlan_depth = ETH_HLEN;

	}

	do {

		struct vlan_hdr vhdr, *vh;



		vh = skb_header_pointer(skb, mac_offset + vlan_depth,

					sizeof(vhdr), &vhdr);

		if (unlikely(!vh || !--parse_depth))

			return (struct vlan_type_depth) { 0 };



		type = vh->h_vlan_encapsulated_proto;

		vlan_depth += VLAN_HLEN;

	} while (eth_type_vlan(type));



	return (struct vlan_type_depth) {

		.type = type,

		.depth = vlan_depth

	};

}

EXPORT_SYMBOL(__vlan_get_protocol_offset);

net/packet/af_packet.c

+3 −2
Original line number Diff line number Diff line
@@ -572,8 +572,9 @@ static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb) 
	__be16 proto = skb->protocol;



	if (unlikely(eth_type_vlan(proto)))

		proto = __vlan_get_protocol_offset(skb, proto,

						   skb_mac_offset(skb), NULL);

		proto = vlan_get_protocol_offset_inline(skb, proto,

							skb_mac_offset(skb),

							NULL);



	return proto;

}