Commit 9d1a8c6f authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'xdp-a-fistful-of-generic-changes-pt-i' 

Alexander Lobakin says:

====================
xdp: a fistful of generic changes pt. I

XDP for idpf is currently 6 chapters:
* convert Rx to libeth;
* convert Tx and stats to libeth;
* generic XDP and XSk code changes (you are here);
* generic XDP and XSk code additions;
* actual XDP for idpf via new libeth_xdp;
* XSk for idpf (via ^).

Part III does the following:
* improve &xdp_buff_xsk cacheline placement;
* does some cleanups with marking read-only bpf_prog and xdp_buff
  arguments const for some generic functions;
* allows attaching already registered XDP memory model to RxQ info;
* makes system percpu page_pools valid XDP memory models;
* starts using netmems in the XDP core code (1 function);
* allows mixing pages from several page_pools within one XDP frame;
* optimizes &xdp_frame layout and removes no-more-used field.

Bullets 4-6 are the most important ones. All of them are prereqs to
libeth_xdp.
====================

Link: https://patch.msgid.link/20241203173733.3181246-1-aleksander.lobakin@intel.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 1daa6591 024bfd2e

include/linux/bpf.h

+6 −6
Original line number Diff line number Diff line
@@ -2591,10 +2591,10 @@ int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 
int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,

			  struct bpf_map *map, bool exclude_ingress);

int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,

			     struct bpf_prog *xdp_prog);

			     const struct bpf_prog *xdp_prog);

int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,

			   struct bpf_prog *xdp_prog, struct bpf_map *map,

			   bool exclude_ingress);

			   const struct bpf_prog *xdp_prog,

			   struct bpf_map *map, bool exclude_ingress);



void __cpu_map_flush(struct list_head *flush_list);

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
@@ -2864,15 +2864,15 @@ struct sk_buff; 


static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,

					   struct sk_buff *skb,

					   struct bpf_prog *xdp_prog)

					   const struct bpf_prog *xdp_prog)

{

	return 0;

}



static inline

int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,

			   struct bpf_prog *xdp_prog, struct bpf_map *map,

			   bool exclude_ingress)

			   const struct bpf_prog *xdp_prog,

			   struct bpf_map *map, bool exclude_ingress)

{

	return 0;

}

include/linux/filter.h

+5 −4
Original line number Diff line number Diff line
@@ -1179,17 +1179,18 @@ static inline int xdp_ok_fwd_dev(const struct net_device *fwd, 
 * This does not appear to be a real limitation for existing software.

 */

int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,

			    struct xdp_buff *xdp, struct bpf_prog *prog);

			    struct xdp_buff *xdp, const struct bpf_prog *prog);

int xdp_do_redirect(struct net_device *dev,

		    struct xdp_buff *xdp,

		    struct bpf_prog *prog);

		    const struct bpf_prog *prog);

int xdp_do_redirect_frame(struct net_device *dev,

			  struct xdp_buff *xdp,

			  struct xdp_frame *xdpf,

			  struct bpf_prog *prog);

			  const struct bpf_prog *prog);

void xdp_do_flush(void);



void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act);

void bpf_warn_invalid_xdp_action(const struct net_device *dev,

				 const struct bpf_prog *prog, u32 act);



#ifdef CONFIG_INET

struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,

include/linux/netdevice.h

+4 −3
Original line number Diff line number Diff line
@@ -3322,6 +3322,7 @@ struct softnet_data { 
};



DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);

DECLARE_PER_CPU(struct page_pool *, system_page_pool);



#ifndef CONFIG_PREEMPT_RT

static inline int dev_recursion_level(void)
@@ -3958,9 +3959,9 @@ static inline void dev_consume_skb_any(struct sk_buff *skb) 
}



u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,

			     struct bpf_prog *xdp_prog);

void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);

int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb);

			     const struct bpf_prog *xdp_prog);

void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog);

int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb);

int netif_rx(struct sk_buff *skb);

int __netif_rx(struct sk_buff *skb);

include/linux/skbuff.h

+1 −1
Original line number Diff line number Diff line
@@ -3627,7 +3627,7 @@ static inline netmem_ref skb_frag_netmem(const skb_frag_t *frag) 
int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,

		    unsigned int headroom);

int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,

			 struct bpf_prog *prog);

			 const struct bpf_prog *prog);



/**

 * skb_frag_address - gets the address of the data contained in a paged fragment

include/net/netmem.h

+76 −2
Original line number Diff line number Diff line
@@ -72,6 +72,22 @@ static inline bool netmem_is_net_iov(const netmem_ref netmem) 
	return (__force unsigned long)netmem & NET_IOV;

}



/**

 * __netmem_to_page - unsafely get pointer to the &page backing @netmem

 * @netmem: netmem reference to convert

 *

 * Unsafe version of netmem_to_page(). When @netmem is always page-backed,

 * e.g. when it's a header buffer, performs faster and generates smaller

 * object code (no check for the LSB, no WARN). When @netmem points to IOV,

 * provokes undefined behaviour.

 *

 * Return: pointer to the &page (garbage if @netmem is not page-backed).

 */

static inline struct page *__netmem_to_page(netmem_ref netmem)

{

	return (__force struct page *)netmem;

}



/* This conversion fails (returns NULL) if the netmem_ref is not struct page

 * backed.

 */
@@ -80,7 +96,7 @@ static inline struct page *netmem_to_page(netmem_ref netmem) 
	if (WARN_ON_ONCE(netmem_is_net_iov(netmem)))

		return NULL;



	return (__force struct page *)netmem;

	return __netmem_to_page(netmem);

}



static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem)
@@ -103,6 +119,17 @@ static inline netmem_ref page_to_netmem(struct page *page) 
	return (__force netmem_ref)page;

}



/**

 * virt_to_netmem - convert virtual memory pointer to a netmem reference

 * @data: host memory pointer to convert

 *

 * Return: netmem reference to the &page backing this virtual address.

 */

static inline netmem_ref virt_to_netmem(const void *data)

{

	return page_to_netmem(virt_to_page(data));

}



static inline int netmem_ref_count(netmem_ref netmem)

{

	/* The non-pp refcount of net_iov is always 1. On net_iov, we only
@@ -127,6 +154,22 @@ static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem) 
	return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV);

}



/**

 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem

 * @netmem: netmem reference to get the pointer from

 *

 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed,

 * e.g. when it's a header buffer, performs faster and generates smaller

 * object code (avoids clearing the LSB). When @netmem points to IOV,

 * provokes invalid memory access.

 *

 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed).

 */

static inline struct page_pool *__netmem_get_pp(netmem_ref netmem)

{

	return __netmem_to_page(netmem)->pp;

}



static inline struct page_pool *netmem_get_pp(netmem_ref netmem)

{

	return __netmem_clear_lsb(netmem)->pp;
@@ -158,12 +201,43 @@ static inline netmem_ref netmem_compound_head(netmem_ref netmem) 
	return page_to_netmem(compound_head(netmem_to_page(netmem)));

}



/**

 * __netmem_address - unsafely get pointer to the memory backing @netmem

 * @netmem: netmem reference to get the pointer for

 *

 * Unsafe version of netmem_address(). When @netmem is always page-backed,

 * e.g. when it's a header buffer, performs faster and generates smaller

 * object code (no check for the LSB). When @netmem points to IOV, provokes

 * undefined behaviour.

 *

 * Return: pointer to the memory (garbage if @netmem is not page-backed).

 */

static inline void *__netmem_address(netmem_ref netmem)

{

	return page_address(__netmem_to_page(netmem));

}



static inline void *netmem_address(netmem_ref netmem)

{

	if (netmem_is_net_iov(netmem))

		return NULL;



	return page_address(netmem_to_page(netmem));

	return __netmem_address(netmem);

}



/**

 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure

 * @netmem: netmem reference to check

 *

 * Return: true if @netmem is page-backed and the page was allocated under

 * memory pressure, false otherwise.

 */

static inline bool netmem_is_pfmemalloc(netmem_ref netmem)

{

	if (netmem_is_net_iov(netmem))

		return false;



	return page_is_pfmemalloc(netmem_to_page(netmem));

}



static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)