Commit ee8f97ef authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski
Browse files

gro_cells: avoid using synchronize_rcu() in gro_cells_destroy() 



Another thing making netns dismantles potentially very slow is located
in gro_cells_destroy(),
whenever cleanup_net() has to remove a device using gro_cells framework.

RTNL is not held at this stage, so synchronize_net()
is calling synchronize_rcu():

netdev_run_todo()
 ip_tunnel_dev_free()
  gro_cells_destroy()
   synchronize_net()
    synchronize_rcu() // Ouch.

This patch uses call_rcu(), and gave me a 25x performance improvement
in my tests.

cleanup_net() is no longer blocked ~10 ms per synchronize_rcu()
call.

In the case we could not allocate the memory needed to queue the
deferred free, use synchronize_rcu_expedited()

v2: made percpu_free_defer_callback() static

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarPaolo Abeni <pabeni@redhat.com>
Link: https://lore.kernel.org/r/20220220041155.607637-1-eric.dumazet@gmail.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent d4276e57

net/core/gro_cells.c

+31 −5
Original line number Diff line number Diff line
@@ -89,8 +89,23 @@ int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) 
}

EXPORT_SYMBOL(gro_cells_init);



struct percpu_free_defer {

	struct rcu_head rcu;

	void __percpu	*ptr;

};



static void percpu_free_defer_callback(struct rcu_head *head)

{

	struct percpu_free_defer *defer;



	defer = container_of(head, struct percpu_free_defer, rcu);

	free_percpu(defer->ptr);

	kfree(defer);

}



void gro_cells_destroy(struct gro_cells *gcells)

{

	struct percpu_free_defer *defer;

	int i;



	if (!gcells->cells)
@@ -102,12 +117,23 @@ void gro_cells_destroy(struct gro_cells *gcells) 
		__netif_napi_del(&cell->napi);

		__skb_queue_purge(&cell->napi_skbs);

	}

	/* This barrier is needed because netpoll could access dev->napi_list

	 * under rcu protection.

	/* We need to observe an rcu grace period before freeing ->cells,

	 * because netpoll could access dev->napi_list under rcu protection.

	 * Try hard using call_rcu() instead of synchronize_rcu(),

	 * because we might be called from cleanup_net(), and we

	 * definitely do not want to block this critical task.

	 */

	synchronize_net();



	defer = kmalloc(sizeof(*defer), GFP_KERNEL | __GFP_NOWARN);

	if (likely(defer)) {

		defer->ptr = gcells->cells;

		call_rcu(&defer->rcu, percpu_free_defer_callback);

	} else {

		/* We do not hold RTNL at this point, synchronize_net()

		 * would not be able to expedite this sync.

		 */

		synchronize_rcu_expedited();

		free_percpu(gcells->cells);

	}

	gcells->cells = NULL;

}

EXPORT_SYMBOL(gro_cells_destroy);