Commit 60c9a34d authored by Konstantin Taranov's avatar Konstantin Taranov Committed by Leon Romanovsky
Browse files

RDMA/mana_ib: add support of multiple ports 



If the HW indicates support of multiple ports for rdma, create an IB device
with a port per netdev in the ethernet mana driver.

CM is only available on port 1, but RC QPs are supported on all
ports.

Signed-off-by: default avatarKonstantin Taranov <kotaranov@microsoft.com>
Link: https://patch.msgid.link/1753174515-23634-1-git-send-email-kotaranov@linux.microsoft.com


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 10d4de41

drivers/infiniband/hw/mana/device.c

+60 −54
Original line number Diff line number Diff line
@@ -77,22 +77,23 @@ static int mana_ib_netdev_event(struct notifier_block *this, 
	struct gdma_context *gc = dev->gdma_dev->gdma_context;

	struct mana_context *mc = gc->mana.driver_data;

	struct net_device *ndev;

	int i;



	/* Only process events from our parent device */

	if (event_dev != mc->ports[0])

		return NOTIFY_DONE;



	for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)

		if (event_dev == mc->ports[i]) {

			switch (event) {

			case NETDEV_CHANGEUPPER:

		ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);

				ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);

				/*

				 * RDMA core will setup GID based on updated netdev.

				 * It's not possible to race with the core as rtnl lock is being

				 * held.

				 */

		ib_device_set_netdev(&dev->ib_dev, ndev, 1);

				ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);



				/* mana_get_primary_netdev() returns ndev with refcount held */

				if (ndev)

					netdev_put(ndev, &dev->dev_tracker);



				return NOTIFY_OK;
@@ -100,6 +101,8 @@ static int mana_ib_netdev_event(struct notifier_block *this, 
				return NOTIFY_DONE;

			}

		}

	return NOTIFY_DONE;

}



static int mana_ib_probe(struct auxiliary_device *adev,

			 const struct auxiliary_device_id *id)
@@ -111,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, 
	struct net_device *ndev;

	struct mana_ib_dev *dev;

	u8 mac_addr[ETH_ALEN];

	int ret;

	int ret, i;



	dev = ib_alloc_device(mana_ib_dev, ib_dev);

	if (!dev)
@@ -126,34 +129,11 @@ static int mana_ib_probe(struct auxiliary_device *adev, 


	if (mana_ib_is_rnic(dev)) {

		dev->ib_dev.phys_port_cnt = 1;

		ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);

		if (!ndev) {

			ret = -ENODEV;

			ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");

			goto free_ib_device;

		}

		ether_addr_copy(mac_addr, ndev->dev_addr);

		addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);

		ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);

		/* mana_get_primary_netdev() returns ndev with refcount held */

		netdev_put(ndev, &dev->dev_tracker);

		if (ret) {

			ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);

			goto free_ib_device;

		}



		dev->nb.notifier_call = mana_ib_netdev_event;

		ret = register_netdevice_notifier(&dev->nb);

		if (ret) {

			ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",

				  ret);

			goto free_ib_device;

		}



		addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);

		ret = mana_ib_gd_query_adapter_caps(dev);

		if (ret) {

			ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);

			goto deregister_net_notifier;

			goto free_ib_device;

		}



		ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
@@ -163,18 +143,44 @@ static int mana_ib_probe(struct auxiliary_device *adev, 
		ret = mana_ib_create_eqs(dev);

		if (ret) {

			ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);

			goto deregister_net_notifier;

			goto free_ib_device;

		}



		ret = mana_ib_gd_create_rnic_adapter(dev);

		if (ret)

			goto destroy_eqs;



		if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)

			dev->ib_dev.phys_port_cnt = mc->num_ports;



		for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {

			ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);

			if (!ndev) {

				ret = -ENODEV;

				ibdev_err(&dev->ib_dev,

					  "Failed to get netdev for IB port %d", i + 1);

				goto destroy_rnic;

			}

			ether_addr_copy(mac_addr, ndev->dev_addr);

			ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);

			/* mana_get_primary_netdev() returns ndev with refcount held */

			netdev_put(ndev, &dev->dev_tracker);

			if (ret) {

				ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);

				goto destroy_rnic;

			}

			ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);

			if (ret) {

				ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);

				goto destroy_rnic;

			}

		}

		dev->nb.notifier_call = mana_ib_netdev_event;

		ret = register_netdevice_notifier(&dev->nb);

		if (ret) {

			ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);

			goto destroy_rnic;

		}

	} else {

		dev->ib_dev.phys_port_cnt = mc->num_ports;

		ret = mana_eth_query_adapter_caps(dev);
@@ -188,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, 
				       MANA_AV_BUFFER_SIZE, 0);

	if (!dev->av_pool) {

		ret = -ENOMEM;

		goto destroy_rnic;

		goto deregister_net_notifier;

	}



	ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -205,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev, 


deallocate_pool:

	dma_pool_destroy(dev->av_pool);

deregister_net_notifier:

	if (mana_ib_is_rnic(dev))

		unregister_netdevice_notifier(&dev->nb);

destroy_rnic:

	if (mana_ib_is_rnic(dev))

		mana_ib_gd_destroy_rnic_adapter(dev);

destroy_eqs:

	if (mana_ib_is_rnic(dev))

		mana_ib_destroy_eqs(dev);

deregister_net_notifier:

	if (mana_ib_is_rnic(dev))

		unregister_netdevice_notifier(&dev->nb);

free_ib_device:

	xa_destroy(&dev->qp_table_wq);

	ib_dealloc_device(&dev->ib_dev);
@@ -227,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev) 
	ib_unregister_device(&dev->ib_dev);

	dma_pool_destroy(dev->av_pool);

	if (mana_ib_is_rnic(dev)) {

		unregister_netdevice_notifier(&dev->nb);

		mana_ib_gd_destroy_rnic_adapter(dev);

		mana_ib_destroy_eqs(dev);

		unregister_netdevice_notifier(&dev->nb);

	}

	xa_destroy(&dev->qp_table_wq);

	ib_dealloc_device(&dev->ib_dev);

drivers/infiniband/hw/mana/main.c

+10 −3
Original line number Diff line number Diff line
@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, 
	immutable->gid_tbl_len = attr.gid_tbl_len;



	if (mana_ib_is_rnic(dev)) {

		if (port_num == 1) {

			immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;

			immutable->max_mad_size = IB_MGMT_MAD_SIZE;

		} else {

			immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP

						    | RDMA_CORE_CAP_ETH_AH;

			immutable->max_mad_size = 0;

		}

	} else {

		immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;

	}
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, 
	props->pkey_tbl_len = 1;

	if (mana_ib_is_rnic(dev)) {

		props->gid_tbl_len = 16;

		props->port_cap_flags = IB_PORT_CM_SUP;

		props->ip_gids = true;

		if (port == 1)

			props->port_cap_flags = IB_PORT_CM_SUP;

	}



	return 0;

drivers/infiniband/hw/mana/mana_ib.h

+1 −0
Original line number Diff line number Diff line
@@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req { 
enum mana_ib_adapter_features {

	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),

	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),

	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),

};



struct mana_ib_query_adapter_caps_resp {