Commit b1d91548 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'net-mlx5e-shampo-fixes-for-64kb-page-size' 

Tariq Toukan says:

====================
net/mlx5e: SHAMPO fixes for 64KB page size

This series by Dragos contains fixes for HW-GRO issues found on systems
with 64KB page size.
====================

Link: https://patch.msgid.link/1762238915-1027590-1-git-send-email-tariqt@nvidia.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents ae4789af d8a7ed95

drivers/net/ethernet/mellanox/mlx5/core/en.h

+3 −0
Original line number Diff line number Diff line
@@ -634,7 +634,10 @@ struct mlx5e_dma_info { 
struct mlx5e_shampo_hd {

	struct mlx5e_frag_page *pages;

	u32 hd_per_wq;

	u32 hd_per_page;

	u16 hd_per_wqe;

	u8 log_hd_per_page;

	u8 log_hd_entry_size;

	unsigned long *bitmap;

	u16 pi;

	u16 ci;

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

+19 −5
Original line number Diff line number Diff line
@@ -791,8 +791,9 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 
				int node)

{

	void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);

	u8 log_hd_per_page, log_hd_entry_size;

	u16 hd_per_wq, hd_per_wqe;

	u32 hd_pool_size;

	u16 hd_per_wq;

	int wq_size;

	int err;
@@ -815,11 +816,24 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 
	if (err)

		goto err_umr_mkey;



	rq->mpwqe.shampo->hd_per_wqe =

		mlx5e_shampo_hd_per_wqe(mdev, params, rqp);

	hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rqp);

	wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));

	hd_pool_size = (rq->mpwqe.shampo->hd_per_wqe * wq_size) /

		MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;



	BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT);

	if (hd_per_wqe >= MLX5E_SHAMPO_WQ_HEADER_PER_PAGE) {

		log_hd_per_page = MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE;

		log_hd_entry_size = MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;

	} else {

		log_hd_per_page = order_base_2(hd_per_wqe);

		log_hd_entry_size = order_base_2(PAGE_SIZE / hd_per_wqe);

	}



	rq->mpwqe.shampo->hd_per_wqe = hd_per_wqe;

	rq->mpwqe.shampo->hd_per_page = BIT(log_hd_per_page);

	rq->mpwqe.shampo->log_hd_per_page = log_hd_per_page;

	rq->mpwqe.shampo->log_hd_entry_size = log_hd_entry_size;



	hd_pool_size = (hd_per_wqe * wq_size) >> log_hd_per_page;



	if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) {

		/* Separate page pool for shampo headers */

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

+39 −33
Original line number Diff line number Diff line
@@ -648,17 +648,20 @@ static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe, 
	umr_wqe->hdr.uctrl.mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);

}



static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq, int header_index)

static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq,

							    int header_index)

{

	BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT);

	struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;



	return &rq->mpwqe.shampo->pages[header_index >> MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE];

	return &shampo->pages[header_index >> shampo->log_hd_per_page];

}



static u64 mlx5e_shampo_hd_offset(int header_index)

static u64 mlx5e_shampo_hd_offset(struct mlx5e_rq *rq, int header_index)

{

	return (header_index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<

		MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;

	struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;

	u32 hd_per_page = shampo->hd_per_page;



	return (header_index & (hd_per_page - 1)) << shampo->log_hd_entry_size;

}



static void mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index);
@@ -671,7 +674,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, 
	u16 pi, header_offset, err, wqe_bbs;

	u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey;

	struct mlx5e_umr_wqe *umr_wqe;

	int headroom, i = 0;

	int headroom, i;



	headroom = rq->buff.headroom;

	wqe_bbs = MLX5E_KSM_UMR_WQEBBS(ksm_entries);
@@ -679,26 +682,25 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, 
	umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);

	build_ksm_umr(sq, umr_wqe, shampo->mkey_be, index, ksm_entries);



	WARN_ON_ONCE(ksm_entries & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1));

	while (i < ksm_entries) {

		struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);

	for (i = 0; i < ksm_entries; i++, index++) {

		struct mlx5e_frag_page *frag_page;

		u64 addr;



		err = mlx5e_page_alloc_fragmented(rq->hd_page_pool, frag_page);

		if (unlikely(err))

		frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);

		header_offset = mlx5e_shampo_hd_offset(rq, index);

		if (!header_offset) {

			err = mlx5e_page_alloc_fragmented(rq->hd_page_pool,

							  frag_page);

			if (err)

				goto err_unmap;

		}



		addr = page_pool_get_dma_addr_netmem(frag_page->netmem);



		for (int j = 0; j < MLX5E_SHAMPO_WQ_HEADER_PER_PAGE; j++) {

			header_offset = mlx5e_shampo_hd_offset(index++);



			umr_wqe->inline_ksms[i++] = (struct mlx5_ksm) {

		umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {

			.key = cpu_to_be32(lkey),

			.va  = cpu_to_be64(addr + header_offset + headroom),

		};

	}

	}



	sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {

		.wqe_type	= MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR,
@@ -713,9 +715,9 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, 
	return 0;



err_unmap:

	while (--i) {

	while (--i >= 0) {

		--index;

		header_offset = mlx5e_shampo_hd_offset(index);

		header_offset = mlx5e_shampo_hd_offset(rq, index);

		if (!header_offset) {

			struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index);
@@ -735,12 +737,11 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) 
	struct mlx5e_icosq *sq = rq->icosq;

	int i, err, max_ksm_entries, len;



	max_ksm_entries = ALIGN_DOWN(MLX5E_MAX_KSM_PER_WQE(rq->mdev),

				     MLX5E_SHAMPO_WQ_HEADER_PER_PAGE);

	max_ksm_entries = MLX5E_MAX_KSM_PER_WQE(rq->mdev);

	ksm_entries = bitmap_find_window(shampo->bitmap,

					 shampo->hd_per_wqe,

					 shampo->hd_per_wq, shampo->pi);

	ksm_entries = ALIGN_DOWN(ksm_entries, MLX5E_SHAMPO_WQ_HEADER_PER_PAGE);

	ksm_entries = ALIGN_DOWN(ksm_entries, shampo->hd_per_page);

	if (!ksm_entries)

		return 0;
@@ -858,7 +859,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) 
{

	struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;



	if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {

	if (((header_index + 1) & (shampo->hd_per_page - 1)) == 0) {

		struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);



		mlx5e_page_release_fragmented(rq->hd_page_pool, frag_page);
@@ -1225,9 +1226,10 @@ static unsigned int mlx5e_lro_update_hdr(struct sk_buff *skb, 
static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)

{

	struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);

	u16 head_offset = mlx5e_shampo_hd_offset(header_index) + rq->buff.headroom;

	u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index);

	void *addr = netmem_address(frag_page->netmem);



	return netmem_address(frag_page->netmem) + head_offset;

	return addr + head_offset + rq->buff.headroom;

}



static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4)
@@ -2267,7 +2269,8 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, 
			  struct mlx5_cqe64 *cqe, u16 header_index)

{

	struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index);

	u16 head_offset = mlx5e_shampo_hd_offset(header_index);

	u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index);

	struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;

	u16 head_size = cqe->shampo.header_size;

	u16 rx_headroom = rq->buff.headroom;

	struct sk_buff *skb = NULL;
@@ -2283,7 +2286,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, 
	data		= hdr + rx_headroom;

	frag_size	= MLX5_SKB_FRAG_SZ(rx_headroom + head_size);



	if (likely(frag_size <= BIT(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE))) {

	if (likely(frag_size <= BIT(shampo->log_hd_entry_size))) {

		/* build SKB around header */

		dma_sync_single_range_for_cpu(rq->pdev, dma_addr, 0, frag_size, rq->buff.map_dir);

		net_prefetchw(hdr);
@@ -2356,6 +2359,9 @@ mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt) 
{

	int nr_frags = skb_shinfo(skb)->nr_frags;



	if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE)

		return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE;

	else

		return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;

}