Commit e534552c authored by Shay Drory's avatar Shay Drory Committed by Saeed Mahameed
Browse files

net/mlx5: Refactor LAG peer device lookout bus logic to mlx5 devcom 



LAG peer device lookout bus logic required the usage of global lock,
mlx5_intf_mutex.
As part of the effort to remove this global lock, refactor LAG peer
device lookout to use mlx5 devcom layer.

Signed-off-by: default avatarShay Drory <shayd@nvidia.com>
Reviewed-by: default avatarMark Bloch <mbloch@nvidia.com>
Reviewed-by: default avatarJacob Keller <jacob.e.keller@intel.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 89d351c2

drivers/net/ethernet/mellanox/mlx5/core/dev.c

+0 −68
Original line number Diff line number Diff line
@@ -566,74 +566,6 @@ bool mlx5_same_hw_devs(struct mlx5_core_dev *dev, struct mlx5_core_dev *peer_dev 
	return (fsystem_guid && psystem_guid && fsystem_guid == psystem_guid);

}



static u32 mlx5_gen_pci_id(const struct mlx5_core_dev *dev)

{

	return (u32)((pci_domain_nr(dev->pdev->bus) << 16) |

		     (dev->pdev->bus->number << 8) |

		     PCI_SLOT(dev->pdev->devfn));

}



static int _next_phys_dev(struct mlx5_core_dev *mdev,

			  const struct mlx5_core_dev *curr)

{

	if (!mlx5_core_is_pf(mdev))

		return 0;



	if (mdev == curr)

		return 0;



	if (!mlx5_same_hw_devs(mdev, (struct mlx5_core_dev *)curr) &&

	    mlx5_gen_pci_id(mdev) != mlx5_gen_pci_id(curr))

		return 0;



	return 1;

}



static void *pci_get_other_drvdata(struct device *this, struct device *other)

{

	if (this->driver != other->driver)

		return NULL;



	return pci_get_drvdata(to_pci_dev(other));

}



static int next_phys_dev_lag(struct device *dev, const void *data)

{

	struct mlx5_core_dev *mdev, *this = (struct mlx5_core_dev *)data;



	mdev = pci_get_other_drvdata(this->device, dev);

	if (!mdev)

		return 0;



	if (!mlx5_lag_is_supported(mdev))

		return 0;



	return _next_phys_dev(mdev, data);

}



static struct mlx5_core_dev *mlx5_get_next_dev(struct mlx5_core_dev *dev,

					       int (*match)(struct device *dev, const void *data))

{

	struct device *next;



	if (!mlx5_core_is_pf(dev))

		return NULL;



	next = bus_find_device(&pci_bus_type, NULL, dev, match);

	if (!next)

		return NULL;



	put_device(next);

	return pci_get_drvdata(to_pci_dev(next));

}



/* Must be called with intf_mutex held */

struct mlx5_core_dev *mlx5_get_next_phys_dev_lag(struct mlx5_core_dev *dev)

{

	lockdep_assert_held(&mlx5_intf_mutex);

	return mlx5_get_next_dev(dev, &next_phys_dev_lag);

}



void mlx5_dev_list_lock(void)

{

	mutex_lock(&mlx5_intf_mutex);

drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c

+8 −4
Original line number Diff line number Diff line
@@ -1212,13 +1212,14 @@ static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, 
	dev->priv.lag = NULL;

}



/* Must be called with intf_mutex held */

/* Must be called with HCA devcom component lock held */

static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)

{

	struct mlx5_devcom_comp_dev *pos = NULL;

	struct mlx5_lag *ldev = NULL;

	struct mlx5_core_dev *tmp_dev;



	tmp_dev = mlx5_get_next_phys_dev_lag(dev);

	tmp_dev = mlx5_devcom_get_next_peer_data(dev->priv.hca_devcom_comp, &pos);

	if (tmp_dev)

		ldev = mlx5_lag_dev(tmp_dev);
@@ -1275,10 +1276,13 @@ void mlx5_lag_add_mdev(struct mlx5_core_dev *dev) 
	if (!mlx5_lag_is_supported(dev))

		return;



	if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))

		return;



recheck:

	mlx5_dev_list_lock();

	mlx5_devcom_comp_lock(dev->priv.hca_devcom_comp);

	err = __mlx5_lag_dev_add_mdev(dev);

	mlx5_dev_list_unlock();

	mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp);



	if (err) {

		msleep(100);

drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c

+14 −0
Original line number Diff line number Diff line
@@ -387,3 +387,17 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, 
	*pos = tmp;

	return data;

}



void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom)

{

	if (IS_ERR_OR_NULL(devcom))

		return;

	down_write(&devcom->comp->sem);

}



void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom)

{

	if (IS_ERR_OR_NULL(devcom))

		return;

	up_write(&devcom->comp->sem);

}

drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h

+4 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
enum mlx5_devcom_component {

	MLX5_DEVCOM_ESW_OFFLOADS,

	MLX5_DEVCOM_MPV,

	MLX5_DEVCOM_HCA_PORTS,

	MLX5_DEVCOM_NUM_COMPONENTS,

};
@@ -52,4 +53,7 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, 
	     data;								  \

	     data = mlx5_devcom_get_next_peer_data_rcu(devcom, &pos))



void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom);

void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom);



#endif /* __LIB_MLX5_DEVCOM_H__ */

drivers/net/ethernet/mellanox/mlx5/core/main.c

+25 −0
Original line number Diff line number Diff line
@@ -73,6 +73,7 @@ 
#include "sf/sf.h"

#include "mlx5_irq.h"

#include "hwmon.h"

#include "lag/lag.h"



MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");

MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
@@ -952,6 +953,27 @@ static void mlx5_pci_close(struct mlx5_core_dev *dev) 
	mlx5_pci_disable_device(dev);

}



static void mlx5_register_hca_devcom_comp(struct mlx5_core_dev *dev)

{

	/* This component is use to sync adding core_dev to lag_dev and to sync

	 * changes of mlx5_adev_devices between LAG layer and other layers.

	 */

	if (!mlx5_lag_is_supported(dev))

		return;



	dev->priv.hca_devcom_comp =

		mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_HCA_PORTS,

					       mlx5_query_nic_system_image_guid(dev),

					       NULL, dev);

	if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))

		mlx5_core_err(dev, "Failed to register devcom HCA component\n");

}



static void mlx5_unregister_hca_devcom_comp(struct mlx5_core_dev *dev)

{

	mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp);

}



static int mlx5_init_once(struct mlx5_core_dev *dev)

{

	int err;
@@ -960,6 +982,7 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) 
	if (IS_ERR(dev->priv.devc))

		mlx5_core_warn(dev, "failed to register devcom device %ld\n",

			       PTR_ERR(dev->priv.devc));

	mlx5_register_hca_devcom_comp(dev);



	err = mlx5_query_board_id(dev);

	if (err) {
@@ -1094,6 +1117,7 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) 
err_irq_cleanup:

	mlx5_irq_table_cleanup(dev);

err_devcom:

	mlx5_unregister_hca_devcom_comp(dev);

	mlx5_devcom_unregister_device(dev->priv.devc);



	return err;
@@ -1123,6 +1147,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) 
	mlx5_events_cleanup(dev);

	mlx5_eq_table_cleanup(dev);

	mlx5_irq_table_cleanup(dev);

	mlx5_unregister_hca_devcom_comp(dev);

	mlx5_devcom_unregister_device(dev->priv.devc);

}