Commit f62a5e71 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'mlx5e-per-queue-coalescing' 

Tariq Toukan says:

====================
mlx5e per-queue coalescing

This patchset adds ethtool per-queue coalescing support for the mlx5e
driver.

The series introduce some changes needed as preparations for the final
patch which adds the support and implements the callbacks.  Main
changes:
- DIM code movements into its own header file.
- Switch to dynamic allocation of the DIM struct in the RQs/SQs.
- Allow coalescing config change without channels reset when possible.
====================

Link: https://lore.kernel.org/r/20240419080445.417574-1-tariqt@nvidia.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents b240fc56 651ebaad

drivers/net/ethernet/mellanox/mlx5/core/en.h

+26 −4
Original line number Diff line number Diff line
@@ -320,6 +320,8 @@ struct mlx5e_params { 
	bool scatter_fcs_en;

	bool rx_dim_enabled;

	bool tx_dim_enabled;

	bool rx_moder_use_cqe_mode;

	bool tx_moder_use_cqe_mode;

	u32 pflags;

	struct bpf_prog *xdp_prog;

	struct mlx5e_xsk *xsk;
@@ -430,7 +432,7 @@ struct mlx5e_txqsq { 
	u16                        cc;

	u16                        skb_fifo_cc;

	u32                        dma_fifo_cc;

	struct dim                 dim; /* Adaptive Moderation */

	struct dim                *dim; /* Adaptive Moderation */



	/* dirtied @xmit */

	u16                        pc ____cacheline_aligned_in_smp;
@@ -722,7 +724,7 @@ struct mlx5e_rq { 
	int                    ix;

	unsigned int           hw_mtu;



	struct dim         dim; /* Dynamic Interrupt Moderation */

	struct dim            *dim; /* Dynamic Interrupt Moderation */



	/* XDP */

	struct bpf_prog __rcu *xdp_prog;
@@ -797,6 +799,10 @@ struct mlx5e_channel { 
	int                        cpu;

	/* Sync between icosq recovery and XSK enable/disable. */

	struct mutex               icosq_recovery_lock;



	/* coalescing configuration */

	struct dim_cq_moder        rx_cq_moder;

	struct dim_cq_moder        tx_cq_moder;

};



struct mlx5e_ptp;
@@ -1040,6 +1046,11 @@ void mlx5e_close_rq(struct mlx5e_rq *rq); 
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter);

void mlx5e_destroy_rq(struct mlx5e_rq *rq);



bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,

			       bool dim_enabled);

bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,

					bool dim_enabled, bool keep_dim_state);



struct mlx5e_sq_param;

int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,

		     struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
@@ -1060,6 +1071,10 @@ int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, 
		  struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,

		  struct mlx5e_cq *cq);

void mlx5e_close_cq(struct mlx5e_cq *cq);

int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,

				u8 cq_period_mode);

int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,

			       u16 cq_period, u16 cq_max_count, u8 cq_period_mode);



int mlx5e_open_locked(struct net_device *netdev);

int mlx5e_close_locked(struct net_device *netdev);
@@ -1118,6 +1133,11 @@ int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 
void mlx5e_tx_err_cqe_work(struct work_struct *recover_work);

void mlx5e_close_txqsq(struct mlx5e_txqsq *sq);



bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,

			       bool dim_enabled);

bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,

					bool dim_enabled, bool keep_dim_state);



static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)

{

	return MLX5_CAP_ETH(mdev, swp) &&
@@ -1179,6 +1199,10 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, 
			       struct ethtool_coalesce *coal,

			       struct kernel_ethtool_coalesce *kernel_coal,

			       struct netlink_ext_ack *extack);

int mlx5e_get_per_queue_coalesce(struct net_device *dev, u32 queue,

				 struct ethtool_coalesce *coal);

int mlx5e_set_per_queue_coalesce(struct net_device *dev, u32 queue,

				 struct ethtool_coalesce *coal);

u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv);

u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv);

int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
@@ -1210,8 +1234,6 @@ int mlx5e_netdev_change_profile(struct mlx5e_priv *priv, 
void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv);

void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);

void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu);

void mlx5e_rx_dim_work(struct work_struct *work);

void mlx5e_tx_dim_work(struct work_struct *work);



void mlx5e_set_xdp_feature(struct net_device *netdev);

netdev_features_t mlx5e_features_check(struct sk_buff *skb,

drivers/net/ethernet/mellanox/mlx5/core/en/channels.c

+83 −0
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 


#include "channels.h"

#include "en.h"

#include "en/dim.h"

#include "en/ptp.h"



unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs)
@@ -55,3 +56,85 @@ bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn) 
	*rqn = c->rq.rqn;

	return true;

}



int mlx5e_channels_rx_change_dim(struct mlx5e_channels *chs, bool enable)

{

	int i;



	for (i = 0; i < chs->num; i++) {

		int err = mlx5e_dim_rx_change(&chs->c[i]->rq, enable);



		if (err)

			return err;

	}



	return 0;

}



int mlx5e_channels_tx_change_dim(struct mlx5e_channels *chs, bool enable)

{

	int i, tc;



	for (i = 0; i < chs->num; i++) {

		for (tc = 0; tc < mlx5e_get_dcb_num_tc(&chs->params); tc++) {

			int err = mlx5e_dim_tx_change(&chs->c[i]->sq[tc], enable);



			if (err)

				return err;

		}

	}



	return 0;

}



int mlx5e_channels_rx_toggle_dim(struct mlx5e_channels *chs)

{

	int i;



	for (i = 0; i < chs->num; i++) {

		/* If dim is enabled for the channel, reset the dim state so the

		 * collected statistics will be reset. This is useful for

		 * supporting legacy interfaces that allow things like changing

		 * the CQ period mode for all channels without disturbing

		 * individual channel configurations.

		 */

		if (chs->c[i]->rq.dim) {

			int err;



			mlx5e_dim_rx_change(&chs->c[i]->rq, false);

			err = mlx5e_dim_rx_change(&chs->c[i]->rq, true);

			if (err)

				return err;

		}

	}



	return 0;

}



int mlx5e_channels_tx_toggle_dim(struct mlx5e_channels *chs)

{

	int i, tc;



	for (i = 0; i < chs->num; i++) {

		for (tc = 0; tc < mlx5e_get_dcb_num_tc(&chs->params); tc++) {

			int err;



			/* If dim is enabled for the channel, reset the dim

			 * state so the collected statistics will be reset. This

			 * is useful for supporting legacy interfaces that allow

			 * things like changing the CQ period mode for all

			 * channels without disturbing individual channel

			 * configurations.

			 */

			if (!chs->c[i]->sq[tc].dim)

				continue;



			mlx5e_dim_tx_change(&chs->c[i]->sq[tc], false);

			err = mlx5e_dim_tx_change(&chs->c[i]->sq[tc], true);

			if (err)

				return err;

		}

	}



	return 0;

}

drivers/net/ethernet/mellanox/mlx5/core/en/channels.h

+4 −0
Original line number Diff line number Diff line
@@ -15,5 +15,9 @@ void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, 
void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn,

				u32 *vhca_id);

bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn);

int mlx5e_channels_rx_change_dim(struct mlx5e_channels *chs, bool enabled);

int mlx5e_channels_tx_change_dim(struct mlx5e_channels *chs, bool enabled);

int mlx5e_channels_rx_toggle_dim(struct mlx5e_channels *chs);

int mlx5e_channels_tx_toggle_dim(struct mlx5e_channels *chs);



#endif /* __MLX5_EN_CHANNELS_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en/dim.h

0 → 100644
+45 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved */



#ifndef __MLX5_EN_DIM_H__

#define __MLX5_EN_DIM_H__



#include <linux/dim.h>

#include <linux/types.h>

#include <linux/mlx5/mlx5_ifc.h>



/* Forward declarations */

struct mlx5e_rq;

struct mlx5e_txqsq;

struct work_struct;



/* convert a boolean value for cqe mode to appropriate dim constant

 * true  : DIM_CQ_PERIOD_MODE_START_FROM_CQE

 * false : DIM_CQ_PERIOD_MODE_START_FROM_EQE

 */

static inline int mlx5e_dim_cq_period_mode(bool start_from_cqe)

{

	return start_from_cqe ? DIM_CQ_PERIOD_MODE_START_FROM_CQE :

		DIM_CQ_PERIOD_MODE_START_FROM_EQE;

}



static inline enum mlx5_cq_period_mode

mlx5e_cq_period_mode(enum dim_cq_period_mode cq_period_mode)

{

	switch (cq_period_mode) {

	case DIM_CQ_PERIOD_MODE_START_FROM_EQE:

		return MLX5_CQ_PERIOD_MODE_START_FROM_EQE;

	case DIM_CQ_PERIOD_MODE_START_FROM_CQE:

		return MLX5_CQ_PERIOD_MODE_START_FROM_CQE;

	default:

		WARN_ON_ONCE(true);

		return MLX5_CQ_PERIOD_MODE_START_FROM_EQE;

	}

}



void mlx5e_rx_dim_work(struct work_struct *work);

void mlx5e_tx_dim_work(struct work_struct *work);

int mlx5e_dim_rx_change(struct mlx5e_rq *rq, bool enabled);

int mlx5e_dim_tx_change(struct mlx5e_txqsq *sq, bool enabled);



#endif /* __MLX5_EN_DIM_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en/params.c

+1 −71
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#include "en/port.h"

#include "en_accel/en_accel.h"

#include "en_accel/ipsec.h"

#include <linux/dim.h>

#include <net/page_pool/types.h>

#include <net/xdp_sock_drv.h>
@@ -513,77 +514,6 @@ int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *param 
	return 0;

}



static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)

{

	struct dim_cq_moder moder = {};



	moder.cq_period_mode = cq_period_mode;

	moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;

	moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;

	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)

		moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;



	return moder;

}



static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)

{

	struct dim_cq_moder moder = {};



	moder.cq_period_mode = cq_period_mode;

	moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;

	moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;

	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)

		moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;



	return moder;

}



static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)

{

	return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?

		DIM_CQ_PERIOD_MODE_START_FROM_CQE :

		DIM_CQ_PERIOD_MODE_START_FROM_EQE;

}



void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode)

{

	if (params->tx_dim_enabled) {

		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);



		params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);

	} else {

		params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);

	}

}



void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode)

{

	if (params->rx_dim_enabled) {

		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);



		params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);

	} else {

		params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);

	}

}



void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)

{

	mlx5e_reset_tx_moderation(params, cq_period_mode);

	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,

			params->tx_cq_moderation.cq_period_mode ==

				MLX5_CQ_PERIOD_MODE_START_FROM_CQE);

}



void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)

{

	mlx5e_reset_rx_moderation(params, cq_period_mode);

	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,

			params->rx_cq_moderation.cq_period_mode ==

				MLX5_CQ_PERIOD_MODE_START_FROM_CQE);

}



bool slow_pci_heuristic(struct mlx5_core_dev *mdev)

{

	u32 link_speed = 0;