Commit c62fdf5b authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: rps: add rps_input_queue_head_add() helper 



process_backlog() can batch increments of sd->input_queue_head,
saving some memory bandwidth.

Also add READ_ONCE()/WRITE_ONCE() annotations around
sd->input_queue_head accesses.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 36b83ffc

include/net/rps.h

+7 −2
Original line number Diff line number Diff line
@@ -138,11 +138,16 @@ static inline void rps_input_queue_tail_save(u32 *dest, u32 tail) 
#endif

}



static inline void rps_input_queue_head_incr(struct softnet_data *sd)

static inline void rps_input_queue_head_add(struct softnet_data *sd, int val)

{

#ifdef CONFIG_RPS

	sd->input_queue_head++;

	WRITE_ONCE(sd->input_queue_head, sd->input_queue_head + val);

#endif

}



static inline void rps_input_queue_head_incr(struct softnet_data *sd)

{

	rps_input_queue_head_add(sd, 1);

}



#endif /* _NET_RPS_H */

net/core/dev.c

+8 −5
Original line number Diff line number Diff line
@@ -4528,7 +4528,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 
	out:

#endif

		rflow->last_qtail =

			per_cpu(softnet_data, next_cpu).input_queue_head;

			READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);

	}



	rflow->cpu = next_cpu;
@@ -4610,7 +4610,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 
		 */

		if (unlikely(tcpu != next_cpu) &&

		    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||

		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -

		     ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -

		      READ_ONCE(rflow->last_qtail))) >= 0)) {

			tcpu = next_cpu;

			rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
@@ -4665,7 +4665,7 @@ bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 
		rflow = &flow_table->flows[flow_id];

		cpu = READ_ONCE(rflow->cpu);

		if (rflow->filter == filter_id && cpu < nr_cpu_ids &&

		    ((int)(per_cpu(softnet_data, cpu).input_queue_head -

		    ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) -

			   READ_ONCE(rflow->last_qtail)) <

		     (int)(10 * flow_table->mask)))

			expire = false;
@@ -6045,9 +6045,10 @@ static int process_backlog(struct napi_struct *napi, int quota) 
			rcu_read_lock();

			__netif_receive_skb(skb);

			rcu_read_unlock();

			rps_input_queue_head_incr(sd);

			if (++work >= quota)

			if (++work >= quota) {

				rps_input_queue_head_add(sd, work);

				return work;

			}



		}
@@ -6070,6 +6071,8 @@ static int process_backlog(struct napi_struct *napi, int quota) 
		backlog_unlock_irq_enable(sd);

	}



	if (work)

		rps_input_queue_head_add(sd, work);

	return work;

}