Commit 7197e080 authored by Steffen Klassert's avatar Steffen Klassert
Browse files

Merge branch 'xfrm: IPsec hardware offload performance improvements' 



Jianbo Liu says:

====================
This patch series optimizes IPsec crypto offload performance by
addressing a lock contention bottleneck using RSS.

The first patch refactors the xfrm_input to avoid a costly
unlock/relock cycle.

The second patch builds on this by removing a redundant replay check,
which is unnecessary for the synchronous hardware path.
====================

Signed-off-by: default avatarSteffen Klassert <steffen.klassert@secunet.com>
parents bfe62db5 b427c0c3

net/xfrm/xfrm_input.c

+15 −15
Original line number Diff line number Diff line
@@ -505,6 +505,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
			async = 1;

			dev_put(skb->dev);

			seq = XFRM_SKB_CB(skb)->seq.input.low;

			spin_lock(&x->lock);

			goto resume;

		}

		/* GRO call */
@@ -541,9 +542,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
				XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);

				goto drop;

			}



			nexthdr = x->type_offload->input_tail(x, skb);

		}



		goto lock;

		goto process;

	}



	family = XFRM_SPI_SKB_CB(skb)->family;
@@ -611,7 +614,12 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
			goto drop;

		}



lock:

process:

		seq_hi = htonl(xfrm_replay_seqhi(x, seq));



		XFRM_SKB_CB(skb)->seq.input.low = seq;

		XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;



		spin_lock(&x->lock);



		if (unlikely(x->km.state != XFRM_STATE_VALID)) {
@@ -638,21 +646,13 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
			goto drop_unlock;

		}



		spin_unlock(&x->lock);



		if (xfrm_tunnel_check(skb, x, family)) {

			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);

			goto drop;

			goto drop_unlock;

		}



		seq_hi = htonl(xfrm_replay_seqhi(x, seq));



		XFRM_SKB_CB(skb)->seq.input.low = seq;

		XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;



		if (crypto_done) {

			nexthdr = x->type_offload->input_tail(x, skb);

		} else {

		if (!crypto_done) {

			spin_unlock(&x->lock);

			dev_hold(skb->dev);



			nexthdr = x->type->input(x, skb);
@@ -660,9 +660,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
				return 0;



			dev_put(skb->dev);

			spin_lock(&x->lock);

		}

resume:

		spin_lock(&x->lock);

		if (nexthdr < 0) {

			if (nexthdr == -EBADMSG) {

				xfrm_audit_state_icvfail(x, skb,
@@ -676,7 +676,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
		/* only the first xfrm gets the encap type */

		encap_type = 0;



		if (xfrm_replay_recheck(x, skb, seq)) {

		if (!crypto_done && xfrm_replay_recheck(x, skb, seq)) {

			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);

			goto drop_unlock;

		}