Commit 67936e13 authored by Tariq Toukan's avatar Tariq Toukan Committed by Saeed Mahameed
Browse files

net/mlx5e: Let channels be SD-aware 



Distribute the channels between the different SD-devices to acheive
local numa node performance on multiple numas.

Each channel works against one specific mdev, creating all datapath
queues against it.

We distribute channels to mdevs in a round-robin policy.

Example for 2 mdevs and 6 channels:
+-------+---------+
| ch ix | mdev ix |
+-------+---------+
|   0   |    0    |
|   1   |    1    |
|   2   |    0    |
|   3   |    1    |
|   4   |    0    |
|   5   |    1    |
+-------+---------+

This round-robin distribution policy is preferred over another suggested
intuitive distribution, in which we first distribute one half of the
channels to mdev #0 and then the second half to mdev #1.

We prefer round-robin for a reason: it is less influenced by changes in
the number of channels. The mapping between channel index and mdev is
fixed, no matter how many channels the user configures. As the channel
stats are persistent to channels closure, changing the mapping every
single time would turn the accumulative stats less representing of the
channel's history.

Per-channel objects should stop using the primary mdev (priv->mdev)
directly, and instead move to using their own channel's mdev.

Signed-off-by: default avatarTariq Toukan <tariqt@nvidia.com>
Reviewed-by: default avatarGal Pressman <gal@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 846122b1

drivers/net/ethernet/mellanox/mlx5/core/en.h

+1 −0
Original line number Diff line number Diff line
@@ -792,6 +792,7 @@ struct mlx5e_channel { 
	struct hwtstamp_config    *tstamp;

	DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);

	int                        ix;

	int                        vec_ix;

	int                        cpu;

	/* Sync between icosq recovery and XSK enable/disable. */

	struct mutex               icosq_recovery_lock;

drivers/net/ethernet/mellanox/mlx5/core/en/params.c

+1 −1
Original line number Diff line number Diff line
@@ -688,7 +688,7 @@ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e 
		.napi = &c->napi,

		.ch_stats = c->stats,

		.node = cpu_to_node(c->cpu),

		.ix = c->ix,

		.ix = c->vec_ix,

	};

}

drivers/net/ethernet/mellanox/mlx5/core/en/qos.c

+4 −4
Original line number Diff line number Diff line
@@ -122,8 +122,8 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, 


	memset(&param_sq, 0, sizeof(param_sq));

	memset(&param_cq, 0, sizeof(param_cq));

	mlx5e_build_sq_param(priv->mdev, params, &param_sq);

	mlx5e_build_tx_cq_param(priv->mdev, params, &param_cq);

	mlx5e_build_sq_param(c->mdev, params, &param_sq);

	mlx5e_build_tx_cq_param(c->mdev, params, &param_cq);

	err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &param_cq, &ccp, &sq->cq);

	if (err)

		goto err_free_sq;
@@ -176,7 +176,7 @@ int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) 
	 */

	smp_wmb();



	qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid);

	qos_dbg(sq->mdev, "Activate QoS SQ qid %u\n", node_qid);

	mlx5e_activate_txqsq(sq);



	return 0;
@@ -190,7 +190,7 @@ void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) 
	if (!sq) /* Handle the case when the SQ failed to open. */

		return;



	qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid);

	qos_dbg(sq->mdev, "Deactivate QoS SQ qid %u\n", qid);

	mlx5e_deactivate_txqsq(sq);



	priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL;

drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c

+2 −2
Original line number Diff line number Diff line
@@ -294,8 +294,8 @@ static void mlx5e_rx_reporter_diagnose_generic_rq(struct mlx5e_rq *rq, 


	params = &priv->channels.params;

	rq_sz = mlx5e_rqwq_get_size(rq);

	real_time =  mlx5_is_real_time_rq(priv->mdev);

	rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL));

	real_time =  mlx5_is_real_time_rq(rq->mdev);

	rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(rq->mdev, params, NULL));



	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ");

	devlink_fmsg_u8_pair_put(fmsg, "type", params->rq_wq_type);

drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c

+1 −2
Original line number Diff line number Diff line
@@ -219,7 +219,6 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, 
						  struct mlx5e_txqsq *sq, int tc)

{

	bool stopped = netif_xmit_stopped(sq->txq);

	struct mlx5e_priv *priv = sq->priv;

	u8 state;

	int err;
@@ -227,7 +226,7 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, 
	devlink_fmsg_u32_pair_put(fmsg, "txq ix", sq->txq_ix);

	devlink_fmsg_u32_pair_put(fmsg, "sqn", sq->sqn);



	err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state);

	err = mlx5_core_query_sq_state(sq->mdev, sq->sqn, &state);

	if (!err)

		devlink_fmsg_u8_pair_put(fmsg, "HW state", state);