Commit 74a8dada authored by Adham Faris's avatar Adham Faris Committed by Saeed Mahameed
Browse files

net/mlx5e: Preparations for supporting larger number of channels 



Data center server CPUs number keeps getting larger with time.
Currently, our driver limits the number of channels to 128.

Maximum channels number is enforced and bounded by hardcoded
defines (en.h/MLX5E_MAX_NUM_CHANNELS) even though the device and machine
(CPUs num) can allow more.

Refactor current implementation in order to handle further channels.

The maximum supported channels number will be increased in the followup
patch.

Introduce RQT size calculation/allocation scheme below:
1) Preserve current RQT size of 256 for channels number up to 128 (the
   old limit).
2) For greater channels number, RQT size is calculated by multiplying
   the channels number by 2 and rounding up the result to the nearest
   power of 2. If the calculated RQT size exceeds the maximum supported
   size by the NIC, fallback to this maximum RQT size
   (1 << log_max_rqt_size).

Since RQT size is no more static, allocate and free the indirection
table SW shadow dynamically.

Signed-off-by: default avatarAdham Faris <afaris@nvidia.com>
Reviewed-by: default avatarTariq Toukan <tariqt@nvidia.com>
Reviewed-by: default avatarJacob Keller <jacob.e.keller@intel.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 0d806cf9

drivers/net/ethernet/mellanox/mlx5/core/en.h

+3 −2
Original line number Diff line number Diff line
@@ -141,7 +141,7 @@ struct page_pool; 
#define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW            0x2



#define MLX5E_MIN_NUM_CHANNELS         0x1

#define MLX5E_MAX_NUM_CHANNELS         (MLX5E_INDIR_RQT_SIZE / 2)

#define MLX5E_MAX_NUM_CHANNELS         128

#define MLX5E_TX_CQ_POLL_BUDGET        128

#define MLX5E_TX_XSK_POLL_BUDGET       64

#define MLX5E_SQ_RECOVER_MIN_INTERVAL  500 /* msecs */
@@ -200,7 +200,8 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) 
{

	return is_kdump_kernel() ?

		MLX5E_MIN_NUM_CHANNELS :

		min_t(int, mlx5_comp_vectors_max(mdev), MLX5E_MAX_NUM_CHANNELS);

		min3(mlx5_comp_vectors_max(mdev), (u32)MLX5E_MAX_NUM_CHANNELS,

		     (u32)(1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)));

}



/* The maximum WQE size can be retrieved by max_wqe_sz_sq in

drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c

+22 −10
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, 
{

	unsigned int i;



	for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++)

	for (i = 0; i < indir->actual_table_size; i++)

		indir->table[i] = i % num_channels;

}
@@ -45,9 +45,9 @@ static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 
}



int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,

			  bool indir_enabled, u32 init_rqn)

			  bool indir_enabled, u32 init_rqn, u32 indir_table_size)

{

	u16 max_size = indir_enabled ? MLX5E_INDIR_RQT_SIZE : 1;

	u16 max_size = indir_enabled ? indir_table_size : 1;



	return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, 1);

}
@@ -68,11 +68,11 @@ static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, unsigned int num_rqns 
{

	unsigned int i;



	for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {

	for (i = 0; i < indir->actual_table_size; i++) {

		unsigned int ix = i;



		if (hfunc == ETH_RSS_HASH_XOR)

			ix = mlx5e_bits_invert(ix, ilog2(MLX5E_INDIR_RQT_SIZE));

			ix = mlx5e_bits_invert(ix, ilog2(indir->actual_table_size));



		ix = indir->table[ix];
@@ -94,7 +94,7 @@ int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 
	u32 *rss_rqns;

	int err;



	rss_rqns = kvmalloc_array(MLX5E_INDIR_RQT_SIZE, sizeof(*rss_rqns), GFP_KERNEL);

	rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);

	if (!rss_rqns)

		return -ENOMEM;
@@ -102,13 +102,25 @@ int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 
	if (err)

		goto out;



	err = mlx5e_rqt_init(rqt, mdev, MLX5E_INDIR_RQT_SIZE, rss_rqns, MLX5E_INDIR_RQT_SIZE);

	err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns,

			     indir->actual_table_size);



out:

	kvfree(rss_rqns);

	return err;

}



#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2



u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)

{

	u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE,

			     roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR));

	u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size);



	return min_t(u32, rqt_size, max_cap_rqt_size);

}



void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt)

{

	mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn);
@@ -151,10 +163,10 @@ int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_ 
	u32 *rss_rqns;

	int err;



	if (WARN_ON(rqt->size != MLX5E_INDIR_RQT_SIZE))

	if (WARN_ON(rqt->size != indir->max_table_size))

		return -EINVAL;



	rss_rqns = kvmalloc_array(MLX5E_INDIR_RQT_SIZE, sizeof(*rss_rqns), GFP_KERNEL);

	rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);

	if (!rss_rqns)

		return -ENOMEM;
@@ -162,7 +174,7 @@ int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_ 
	if (err)

		goto out;



	err = mlx5e_rqt_redirect(rqt, rss_rqns, MLX5E_INDIR_RQT_SIZE);

	err = mlx5e_rqt_redirect(rqt, rss_rqns, indir->actual_table_size);



out:

	kvfree(rss_rqns);

drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h

+6 −3
Original line number Diff line number Diff line
@@ -6,12 +6,14 @@ 


#include <linux/kernel.h>



#define MLX5E_INDIR_RQT_SIZE (1 << 8)

#define MLX5E_INDIR_MIN_RQT_SIZE (BIT(8))



struct mlx5_core_dev;



struct mlx5e_rss_params_indir {

	u32 table[MLX5E_INDIR_RQT_SIZE];

	u32 *table;

	u32 actual_table_size;

	u32 max_table_size;

};



void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir,
@@ -24,7 +26,7 @@ struct mlx5e_rqt { 
};



int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,

			  bool indir_enabled, u32 init_rqn);

			  bool indir_enabled, u32 init_rqn, u32 indir_table_size);

int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,

			 u32 *rqns, unsigned int num_rqns,

			 u8 hfunc, struct mlx5e_rss_params_indir *indir);
@@ -35,6 +37,7 @@ static inline u32 mlx5e_rqt_get_rqtn(struct mlx5e_rqt *rqt) 
	return rqt->rqtn;

}



u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels);

int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn);

int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_rqns,

			     u8 hfunc, struct mlx5e_rss_params_indir *indir);

drivers/net/ethernet/mellanox/mlx5/core/en/rss.c

+90 −18
Original line number Diff line number Diff line
@@ -81,14 +81,75 @@ struct mlx5e_rss { 
	refcount_t refcnt;

};



static struct mlx5e_rss *mlx5e_rss_alloc(void)

void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels)

{

	return kvzalloc(sizeof(struct mlx5e_rss), GFP_KERNEL);

	rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels);

}



static void mlx5e_rss_free(struct mlx5e_rss *rss)

int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev,

				u32 actual_table_size, u32 max_table_size)

{

	indir->table = kvmalloc_array(max_table_size, sizeof(*indir->table), GFP_KERNEL);

	if (!indir->table)

		return -ENOMEM;



	indir->max_table_size = max_table_size;

	indir->actual_table_size = actual_table_size;



	return 0;

}



void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir)

{

	kvfree(indir->table);

}



static int mlx5e_rss_copy(struct mlx5e_rss *to, const struct mlx5e_rss *from)

{

	u32 *dst_indir_table;



	if (to->indir.actual_table_size != from->indir.actual_table_size ||

	    to->indir.max_table_size != from->indir.max_table_size) {

		mlx5e_rss_warn(to->mdev,

			       "Failed to copy RSS due to size mismatch, src (actual %u, max %u) != dst (actual %u, max %u)\n",

			       from->indir.actual_table_size, from->indir.max_table_size,

			       to->indir.actual_table_size, to->indir.max_table_size);

		return -EINVAL;

	}



	dst_indir_table = to->indir.table;

	*to = *from;

	to->indir.table = dst_indir_table;

	memcpy(to->indir.table, from->indir.table,

	       from->indir.actual_table_size * sizeof(*from->indir.table));

	return 0;

}



static struct mlx5e_rss *mlx5e_rss_init_copy(const struct mlx5e_rss *from)

{

	struct mlx5e_rss *rss;

	int err;



	rss = kvzalloc(sizeof(*rss), GFP_KERNEL);

	if (!rss)

		return ERR_PTR(-ENOMEM);



	err = mlx5e_rss_params_indir_init(&rss->indir, from->mdev, from->indir.actual_table_size,

					  from->indir.max_table_size);

	if (err)

		goto err_free_rss;



	err = mlx5e_rss_copy(rss, from);

	if (err)

		goto err_free_indir;



	return rss;



err_free_indir:

	mlx5e_rss_params_indir_cleanup(&rss->indir);

err_free_rss:

	kvfree(rss);

	return ERR_PTR(err);

}



static void mlx5e_rss_params_init(struct mlx5e_rss *rss)
@@ -287,27 +348,35 @@ static int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss) 
	mlx5e_rss_params_init(rss);

	refcount_set(&rss->refcnt, 1);



	return mlx5e_rqt_init_direct(&rss->rqt, rss->mdev, true, rss->drop_rqn);

	return mlx5e_rqt_init_direct(&rss->rqt, rss->mdev, true,

				     rss->drop_rqn, rss->indir.max_table_size);

}



struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn,

				 const struct mlx5e_packet_merge_param *init_pkt_merge_param,

				 enum mlx5e_rss_init_type type)

				 enum mlx5e_rss_init_type type, unsigned int nch,

				 unsigned int max_nch)

{

	struct mlx5e_rss *rss;

	int err;



	rss = mlx5e_rss_alloc();

	rss = kvzalloc(sizeof(*rss), GFP_KERNEL);

	if (!rss)

		return ERR_PTR(-ENOMEM);



	err = mlx5e_rss_params_indir_init(&rss->indir, mdev,

					  mlx5e_rqt_size(mdev, nch),

					  mlx5e_rqt_size(mdev, max_nch));

	if (err)

		goto err_free_rss;



	rss->mdev = mdev;

	rss->inner_ft_support = inner_ft_support;

	rss->drop_rqn = drop_rqn;



	err = mlx5e_rss_init_no_tirs(rss);

	if (err)

		goto err_free_rss;

		goto err_free_indir;



	if (type == MLX5E_RSS_INIT_NO_TIRS)

		goto out;
@@ -329,8 +398,10 @@ struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_suppo 
	mlx5e_rss_destroy_tirs(rss, false);

err_destroy_rqt:

	mlx5e_rqt_destroy(&rss->rqt);

err_free_indir:

	mlx5e_rss_params_indir_cleanup(&rss->indir);

err_free_rss:

	mlx5e_rss_free(rss);

	kvfree(rss);

	return ERR_PTR(err);

}
@@ -345,7 +416,8 @@ int mlx5e_rss_cleanup(struct mlx5e_rss *rss) 
		mlx5e_rss_destroy_tirs(rss, true);



	mlx5e_rqt_destroy(&rss->rqt);

	mlx5e_rss_free(rss);

	mlx5e_rss_params_indir_cleanup(&rss->indir);

	kvfree(rss);



	return 0;

}
@@ -482,7 +554,7 @@ int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc) 
{

	if (indir)

		memcpy(indir, rss->indir.table,

		       MLX5E_INDIR_RQT_SIZE * sizeof(*rss->indir.table));

		       rss->indir.actual_table_size * sizeof(*rss->indir.table));



	if (key)

		memcpy(key, rss->hash.toeplitz_hash_key,
@@ -503,11 +575,9 @@ int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, 
	struct mlx5e_rss *old_rss;

	int err = 0;



	old_rss = mlx5e_rss_alloc();

	if (!old_rss)

		return -ENOMEM;



	*old_rss = *rss;

	old_rss = mlx5e_rss_init_copy(rss);

	if (IS_ERR(old_rss))

		return PTR_ERR(old_rss);



	if (hfunc && *hfunc != rss->hash.hfunc) {

		switch (*hfunc) {
@@ -534,13 +604,13 @@ int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, 
		changed_indir = true;



		memcpy(rss->indir.table, indir,

		       MLX5E_INDIR_RQT_SIZE * sizeof(*rss->indir.table));

		       rss->indir.actual_table_size * sizeof(*rss->indir.table));

	}



	if (changed_indir && rss->enabled) {

		err = mlx5e_rss_apply(rss, rqns, num_rqns);

		if (err) {

			*rss = *old_rss;

			mlx5e_rss_copy(rss, old_rss);

			goto out;

		}

	}
@@ -549,7 +619,9 @@ int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, 
		mlx5e_rss_update_tirs(rss);



out:

	mlx5e_rss_free(old_rss);

	mlx5e_rss_params_indir_cleanup(&old_rss->indir);

	kvfree(old_rss);



	return err;

}

drivers/net/ethernet/mellanox/mlx5/core/en/rss.h

+6 −1
Original line number Diff line number Diff line
@@ -18,9 +18,14 @@ mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt); 


struct mlx5e_rss;



int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev,

				u32 actual_table_size, u32 max_table_size);

void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir);

void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels);

struct mlx5e_rss *mlx5e_rss_init(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn,

				 const struct mlx5e_packet_merge_param *init_pkt_merge_param,

				 enum mlx5e_rss_init_type type);

				 enum mlx5e_rss_init_type type, unsigned int nch,

				 unsigned int max_nch);

int mlx5e_rss_cleanup(struct mlx5e_rss *rss);



void mlx5e_rss_refcnt_inc(struct mlx5e_rss *rss);