Commit 8b0e64d6 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'enic-sr-iov-v2-preparatory-infrastructure' 

Satish Kharat says:

====================
enic: SR-IOV V2 preparatory infrastructure

This is the first of four series adding SR-IOV V2 support to the enic
driver for Cisco VIC 14xx/15xx adapters.

The existing V1 SR-IOV implementation has VFs that interact directly
with the VIC firmware, leaving the PF driver with no visibility or
control over VF behavior. V2 introduces a PF-mediated model where VFs
communicate with the PF through a mailbox over a dedicated admin
channel. This brings enic in line with the standard Linux SR-IOV
model, enabling full PF management of VFs via ip link (MAC, VLAN,
link state, spoofchk, trust, and per-VF statistics).

This preparatory series adds detection and resource helper code with
no functional change to existing driver behavior:

  - Extend BAR resource discovery for admin channel resources
  - Register the V2 VF PCI device ID
  - Detect VF type (V1/V2/usNIC) from SR-IOV PCI capability
  - Make enic_dev_enable/disable ref-counted for shared use by data
    path and admin channel
  - Add type-aware resource allocation for admin WQ/RQ/CQ/INTR
  - Detect presence of admin channel resources at probe time

Tested on VIC 14xx and 15xx series adapters with V2 VFs under KVM
(sriov_numvfs, VF passthrough, ip link VF configuration, VF traffic).

Based in part on initial work by Christian Benvenuti.
====================

Link: https://patch.msgid.link/20260401-enic-sriov-v2-prep-v4-0-d5834b2ef1b9@cisco.com


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 0ea7e61f 4368f5fa

drivers/net/ethernet/cisco/enic/enic.h

+18 −0
Original line number Diff line number Diff line
@@ -225,6 +225,13 @@ struct enic_rq { 
	struct page_pool *pool;

} ____cacheline_aligned;



enum enic_vf_type {

	ENIC_VF_TYPE_NONE,

	ENIC_VF_TYPE_V1,

	ENIC_VF_TYPE_USNIC,

	ENIC_VF_TYPE_V2,

};



/* Per-instance private data structure */

struct enic {

	struct net_device *netdev;
@@ -252,6 +259,8 @@ struct enic { 
#ifdef CONFIG_PCI_IOV

	u16 num_vfs;

#endif

	enum enic_vf_type vf_type;

	unsigned int enable_count;

	spinlock_t enic_api_lock;

	bool enic_api_busy;

	struct enic_port_profile *pp;
@@ -280,6 +289,13 @@ struct enic { 
	u8 rss_key[ENIC_RSS_LEN];

	struct vnic_gen_stats gen_stats;

	enum ext_cq ext_cq;



	/* Admin channel resources for SR-IOV MBOX */

	bool has_admin_channel;

	struct vnic_wq admin_wq;

	struct vnic_rq admin_rq;

	struct vnic_cq admin_cq[2];

	struct vnic_intr admin_intr;

};



static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev)
@@ -297,6 +313,8 @@ static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev) 
	dev_warn(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)

#define vdev_info(vdev, fmt, ...)					\

	dev_info(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)

#define vdev_dbg(vdev, fmt, ...)					\

	dev_dbg(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)



#define vdev_neterr(vdev, fmt, ...)					\

	netdev_err(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)

drivers/net/ethernet/cisco/enic/enic_dev.c

+13 −4
Original line number Diff line number Diff line
@@ -131,10 +131,13 @@ int enic_dev_set_ig_vlan_rewrite_mode(struct enic *enic) 


int enic_dev_enable(struct enic *enic)

{

	int err;

	int err = 0;



	spin_lock_bh(&enic->devcmd_lock);

	if (enic->enable_count == 0)

		err = vnic_dev_enable_wait(enic->vdev);

	if (!err)

		enic->enable_count++;

	spin_unlock_bh(&enic->devcmd_lock);



	return err;
@@ -142,9 +145,15 @@ int enic_dev_enable(struct enic *enic) 


int enic_dev_disable(struct enic *enic)

{

	int err;

	int err = 0;



	spin_lock_bh(&enic->devcmd_lock);

	if (enic->enable_count == 0) {

		spin_unlock_bh(&enic->devcmd_lock);

		return 0;

	}

	enic->enable_count--;

	if (enic->enable_count == 0)

		err = vnic_dev_disable(enic->vdev);

	spin_unlock_bh(&enic->devcmd_lock);

drivers/net/ethernet/cisco/enic/enic_main.c

+57 −2
Original line number Diff line number Diff line
@@ -66,12 +66,15 @@ 
#define PCI_DEVICE_ID_CISCO_VIC_ENET         0x0043  /* ethernet vnic */

#define PCI_DEVICE_ID_CISCO_VIC_ENET_DYN     0x0044  /* enet dynamic vnic */

#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF      0x0071  /* enet SRIOV VF */

#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2   0x02b7  /* enet SRIOV V2 VF */

#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF_USNIC 0x00cf /* enet USNIC VF */



/* Supported devices */

static const struct pci_device_id enic_id_table[] = {

	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET) },

	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_DYN) },

	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF) },

	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2) },

	{ 0, }	/* end of table */

};
@@ -307,7 +310,8 @@ int enic_sriov_enabled(struct enic *enic) 


static int enic_is_sriov_vf(struct enic *enic)

{

	return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;

	return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF ||

	       enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2;

}



int enic_is_valid_vf(struct enic *enic, int vf)
@@ -1746,7 +1750,11 @@ static int enic_open(struct net_device *netdev) 
	if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)

		for (i = 0; i < enic->wq_count; i++)

			napi_enable(&enic->napi[enic_cq_wq(enic, i)]);

	enic_dev_enable(enic);

	err = enic_dev_enable(enic);

	if (err) {

		netdev_err(netdev, "Failed to enable device: %d\n", err);

		goto err_out_dev_enable;

	}



	for (i = 0; i < enic->intr_count; i++)

		vnic_intr_unmask(&enic->intr[i]);
@@ -1756,6 +1764,17 @@ static int enic_open(struct net_device *netdev) 


	return 0;



err_out_dev_enable:

	for (i = 0; i < enic->rq_count; i++)

		napi_disable(&enic->napi[i]);

	if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)

		for (i = 0; i < enic->wq_count; i++)

			napi_disable(&enic->napi[enic_cq_wq(enic, i)]);

	netif_tx_disable(netdev);

	if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic))

		enic_dev_del_station_addr(enic);

	for (i = 0; i < enic->wq_count; i++)

		vnic_wq_disable(&enic->wq[i].vwq);

err_out_free_rq:

	for (i = 0; i < enic->rq_count; i++) {

		ret = vnic_rq_disable(&enic->rq[i].vrq);
@@ -2618,6 +2637,41 @@ static void enic_iounmap(struct enic *enic) 
			iounmap(enic->bar[i].vaddr);

}



#ifdef CONFIG_PCI_IOV

static void enic_sriov_detect_vf_type(struct enic *enic)

{

	struct pci_dev *pdev = enic->pdev;

	int pos;

	u16 vf_dev_id;



	if (enic_is_sriov_vf(enic) || enic_is_dynamic(enic))

		return;



	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);

	if (!pos) {

		enic->vf_type = ENIC_VF_TYPE_NONE;

		return;

	}



	pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &vf_dev_id);



	switch (vf_dev_id) {

	case PCI_DEVICE_ID_CISCO_VIC_ENET_VF:

		enic->vf_type = ENIC_VF_TYPE_V1;

		break;

	case PCI_DEVICE_ID_CISCO_VIC_ENET_VF_USNIC:

		enic->vf_type = ENIC_VF_TYPE_USNIC;

		break;

	case PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2:

		enic->vf_type = ENIC_VF_TYPE_V2;

		break;

	default:

		enic->vf_type = ENIC_VF_TYPE_NONE;

		break;

	}

}

#endif



static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)

{

	struct device *dev = &pdev->dev;
@@ -2731,6 +2785,7 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 
			num_pps = enic->num_vfs;

		}

	}

	enic_sriov_detect_vf_type(enic);

#endif



	/* Allocate structure for port profiles */

drivers/net/ethernet/cisco/enic/enic_res.c

+10 −2
Original line number Diff line number Diff line
@@ -205,10 +205,18 @@ void enic_get_res_counts(struct enic *enic) 
	enic->cq_count = enic->cq_avail;

	enic->intr_count = enic->intr_avail;



	enic->has_admin_channel =

		vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_WQ) >= 1 &&

		vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_RQ) >= 1 &&

		vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_CQ) >=

			ARRAY_SIZE(enic->admin_cq) &&

		vnic_dev_get_res_count(enic->vdev, RES_TYPE_SRIOV_INTR) >= 1;



	dev_info(enic_get_dev(enic),

		"vNIC resources avail: wq %d rq %d cq %d intr %d\n",

		"vNIC resources avail: wq %d rq %d cq %d intr %d admin %s\n",

		enic->wq_avail, enic->rq_avail,

		enic->cq_avail, enic->intr_avail);

		enic->cq_avail, enic->intr_avail,

		enic->has_admin_channel ? "yes" : "no");

}



void enic_init_vnic_resources(struct enic *enic)

drivers/net/ethernet/cisco/enic/vnic_cq.c

+11 −3
Original line number Diff line number Diff line
@@ -20,13 +20,14 @@ void vnic_cq_free(struct vnic_cq *cq) 
	cq->ctrl = NULL;

}



int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index,

	unsigned int desc_count, unsigned int desc_size)

int vnic_cq_alloc_with_type(struct vnic_dev *vdev, struct vnic_cq *cq,

			    unsigned int index, unsigned int desc_count,

			    unsigned int desc_size, unsigned int res_type)

{

	cq->index = index;

	cq->vdev = vdev;



	cq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_CQ, index);

	cq->ctrl = vnic_dev_get_res(vdev, res_type, index);

	if (!cq->ctrl) {

		vdev_err(vdev, "Failed to hook CQ[%d] resource\n", index);

		return -EINVAL;
@@ -35,6 +36,13 @@ int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, 
	return vnic_dev_alloc_desc_ring(vdev, &cq->ring, desc_count, desc_size);

}



int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index,

		  unsigned int desc_count, unsigned int desc_size)

{

	return vnic_cq_alloc_with_type(vdev, cq, index, desc_count, desc_size,

				       RES_TYPE_CQ);

}



void vnic_cq_init(struct vnic_cq *cq, unsigned int flow_control_enable,

	unsigned int color_enable, unsigned int cq_head, unsigned int cq_tail,

	unsigned int cq_tail_color, unsigned int interrupt_enable,